- developers-list - monetdb.org

Re: [Monetdb-developers] [Monetdb-checkins] MonetDB5/src/modules/mal rdf.mx, , 1.1, 1.2
by Romulo Goncalves 01 Apr '08

01 Apr '08

Marco Antonelli wrote: > Update of /cvsroot/monetdb/MonetDB5/src/modules/mal > In directory sc8-pr-cvs16.sourceforge.net:/tmp/cvs-serv9457/src/modules/mal > > Modified Files: > rdf.mx > Log Message: The log message is missing... It is always nice to have a summary of what you are check in... It also helps the developers in future debugging tasks.... Romulo > > > U rdf.mx > Index: rdf.mx > =================================================================== > RCS file: /cvsroot/monetdb/MonetDB5/src/modules/mal/rdf.mx,v > retrieving revision 1.1 > retrieving revision 1.2 > diff -u -d -r1.1 -r1.2 > --- rdf.mx 26 Mar 2008 12:51:06 -0000 1.1 > +++ rdf.mx 1 Apr 2008 14:48:08 -0000 1.2 > @@ -41,7 +41,7 @@ > address RDFimportGraph > comment "import a graph in a named rdf db"; > > -command exportGraph( url:str, db:str, graph:str) : void > +command exportGraph( url:str, db:str, graph:str, order:str, syntax:str) : void > address RDFexportGraph > comment "export a graph in a named rdf db"; > > @@ -62,6 +62,7 @@ > #include <raptor.h> > > #define rdf_export extern > +#define RET_MAXLEN 2048 > > typedef struct { > lng *cur; > @@ -73,7 +74,7 @@ > oid *o; > oid *p; > oid *s; > - str ret; > + str *ret; > } triple_handler_vars; > //TODO > //static lng RDFestimate(str url) { > @@ -91,34 +92,31 @@ > > void message_handler(void *user_data, raptor_locator *locator, const char *message) > { > - unsigned int maxlength = 2048; > - char finalmessage[maxlength]; > size_t length = 0; > - char *msg = finalmessage; > str *ret = (str *)user_data; > + str msg = *ret; > if (locator->uri) > { > - snprintf(msg, maxlength, "In URI location \"%s\": ", raptor_uri_as_string(locator->uri)); > - length = strlen(finalmessage); > - msg = finalmessage + length*(sizeof(char)); > + snprintf(msg, RET_MAXLEN, "In URI location \"%s\": ", raptor_uri_as_string(locator->uri)); > + length = strlen(*ret); > + msg = *ret + length * (sizeof(char)); > } > - if (locator->line >=0 && length < maxlength-1) > + if (locator->line >=0 && length < RET_MAXLEN-1) > { > - snprintf(msg, maxlength-length, "at line %d: ", locator->line); > - length = strlen(finalmessage); > - msg = finalmessage + length*(sizeof(char)); > + snprintf(msg, RET_MAXLEN-length, "at line %d: ", locator->line); > + length = strlen(*ret); > + msg = *ret + length * (sizeof(char)); > } > - if (locator->column >=0 && length < maxlength-1) > + if (locator->column >=0 && length < RET_MAXLEN-1) > { > - snprintf(msg, maxlength-length, "at column %d: ", locator->column); > - length = strlen(finalmessage); > - msg = finalmessage + length*(sizeof(char)); > + snprintf(msg, RET_MAXLEN-length, "at column %d: ", locator->column); > + length = strlen(*ret); > + msg = *ret + length * (sizeof(char)); > } > - if (message && length < maxlength-1) > + if (message && length < RET_MAXLEN-1) > { > - snprintf(msg, maxlength-length, "%s\n", message); > + snprintf(msg, RET_MAXLEN-length, "%s\n", message); > } > - *ret=finalmessage; > } > > @= RDFins > @@ -133,7 +131,7 @@ > bun = (BUN) map->batCount; > if (!BUNappend(map,(ptr)@1v, TRUE)) { > GDKerror("RDFimport: @1[" LLFMT "] = %s, BUNappend fails\n", cur, @1v); > - data->ret = RDFimport_fail(map,ob,pb,sb); > + *(data->ret) = RDFimport_fail(map,ob,pb,sb); > } > } > /* s,p,o[cur] are set to the virtual oid which is bun s p and o are arrays of oid */ > @@ -153,9 +151,10 @@ > oid *o = data->o; > oid *p = data->p; > oid *s = data->s; > + str *ret = data->ret; > BATiter mi = bat_iterator(BATmirror(map)); > BUN bun; > - if (data->ret!=MAL_SUCCEED) > + if (strlen(*ret)!=0) > return; > /* get the subject */ > if (triple->subject_type == RAPTOR_IDENTIFIER_TYPE_RESOURCE) > @@ -164,7 +163,7 @@ > sv = (str)triple->subject; > else > { > - data->ret="Unexpected subject type"; > + *ret="Unexpected subject type"; > return; > } > > @@ -173,7 +172,7 @@ > pv = (str)raptor_uri_as_string((raptor_uri *)triple->predicate); > else > { > - data->ret="Unexpected predicate type"; > + *ret="Unexpected predicate type"; > return; > } > > @@ -190,18 +189,18 @@ > ov = (str)raptor_uri_as_string((raptor_uri*)triple->object); > else > { > - data->ret="Unexpected object type"; > + *ret="Unexpected object type"; > return; > } > > @:RDFins(o)@ > - if (data->ret == MAL_SUCCEED) > + if (strlen(*ret)==0) > { > @:RDFins(p)@ > - if (data->ret == MAL_SUCCEED) > + if (strlen(*ret)==0) > { > @:RDFins(s)@ > - if (data->ret == MAL_SUCCEED) > + if (strlen(*ret)==0) > { > if (++(*cur) > *lim) { > *lim *= 2; > @@ -209,7 +208,7 @@ > pb = BATextend(pb, *lim); > sb = BATextend(sb, *lim); > if (ob == NULL || pb == NULL || sb == NULL) > - data->ret = RDFimport_fail(map,ob,pb,sb); > + *ret = RDFimport_fail(map,ob,pb,sb); > o = (oid *) Tloc(ob, BUNfirst(ob)); > p = (oid *) Tloc(pb, BUNfirst(pb)); > s = (oid *) Tloc(sb, BUNfirst(sb)); > @@ -223,6 +222,7 @@ > RDFimport_(BAT** S, BAT** P, BAT **O, str url, BAT *map) { > raptor_parser *parser; > raptor_uri *rdf_file; > + str ret; > raptor_init(); > parser = raptor_new_parser("rdfxml"); > if (parser) > @@ -244,41 +244,15 @@ > p = (oid *) Tloc(pb, BUNfirst(pb)); > s = (oid *) Tloc(sb, BUNfirst(sb)); > > - //char obuf[20],*op, pbuf[20],*pp, sbuf[20],*sp; > - //str ov, pv, sv; > - > if (map == NULL || ob == NULL || pb == NULL || sb == NULL) > return RDFimport_fail(map,ob,pb,sb); > - > - > - > -// strcpy(obuf, "object"); for(ov=op=obuf; *op; op++); op[1]=0; > -// strcpy(pbuf, "predicate"); for(pv=pp=pbuf; *pp; pp++); pp[1]=0; > -// strcpy(sbuf, "subject"); for(sv=sp=sbuf; *sp; sp++); sp[1]=0; > -// while (cur < 8 /*redland gives me (ov,pv,sv)*/ ) { > -// *op = (cur & 4)?'1':'0'; > -// *pp = (cur & 2)?'1':'0'; > -// *sp = (cur & 1)?'1':'0'; > -// RDFins(o) /* mx macros*/ > -// RDFins(p) > -// RDFins(s) > -// > -// if (++cur > lim) { > -// lim *= 2; > -// ob = BATextend(ob, lim); > -// pb = BATextend(pb, lim); > -// sb = BATextend(sb, lim); > -// if (ob == NULL || pb == NULL || sb == NULL) > -// return RDFimport_fail(map,ob,pb,sb); > -// o = (oid *) Tloc(ob, BUNfirst(ob)); > -// p = (oid *) Tloc(pb, BUNfirst(pb)); > -// s = (oid *) Tloc(sb, BUNfirst(sb)); > -// } > -// } > > - data = malloc(sizeof(triple_handler_vars)); > + data = GDKmalloc(sizeof(triple_handler_vars)); > if (data) > { > + char ret_array[RET_MAXLEN]; > + str ret_ptr = ret_array; > + ret_array[0] = '\0'; > data->cur = cur; > data->lim = lim; > data->map =map; > @@ -288,18 +262,18 @@ > data->s = s; > data->p = p; > data->o = o; > - data->ret = MAL_SUCCEED; > + data->ret = &ret_ptr; > raptor_set_statement_handler(parser, data, triples_handler); > - raptor_set_fatal_error_handler(parser, &data->ret, message_handler); > - raptor_set_error_handler(parser, &data->ret, message_handler); > - raptor_set_warning_handler(parser, &data->ret, message_handler); > - //raptor_set_parser_strict(parser, TRUE); > + raptor_set_fatal_error_handler(parser, &ret_ptr, message_handler); > + raptor_set_error_handler(parser, &ret_ptr, message_handler); > + raptor_set_warning_handler(parser, &ret_ptr, message_handler); > + raptor_set_parser_strict(parser, TRUE); > > if (raptor_parse_file(parser, rdf_file, NULL) == 0) > { > - /* do something with redland */ > - if (data->ret == MAL_SUCCEED) > + if(strlen(ret_ptr)==0) > { > + ret = MAL_SUCCEED; > BATseqbase(ob, 0); > BATseqbase(pb, 0); > BATseqbase(sb, 0); > @@ -312,30 +286,31 @@ > BATkey(BATmirror(ob), FALSE); > BATkey(BATmirror(sb), FALSE); > BATkey(BATmirror(pb), FALSE); > - > - free(data); > - raptor_free_parser(parser); > - raptor_free_uri(rdf_file); > - raptor_finish(); > - return MAL_SUCCEED; > } > - else > - throw(MAL, "rdf.import", data->ret); > } > else > - throw(MAL, "rdf.import", "RDF parsing error"); > + ret = "RDF parsing error"; > + GDKfree(data); > } > else > - throw(MAL, "rdf.import", "triple_handler_vars malloc failed"); > + ret = "triple_handler_vars malloc failed"; > + raptor_free_uri(rdf_file); > } > else > - throw(MAL, "rdf.import", "Unexistent file: \"%s\"", url); > + ret = "Unexistent url"; > + raptor_free_parser(parser); > } > else > - throw(MAL, "rdf.import", "Could not create a Redland parser"); > + ret = "Could not create a Redland parser"; > + raptor_finish(); > + if (ret != MAL_SUCCEED) > + throw(MAL, "rdf.import", ret); > + > + return MAL_SUCCEED; > } > > -rdf_export str RDFimport(int* S, int* P, int* O, str *url, bat m) > +rdf_export > +str RDFimport(int* S, int* P, int* O, str *url, bat m) > { > BAT *sb = NULL, *ob=NULL, *pb = NULL, *map = BATdescriptor(m); > str ret; > @@ -352,45 +327,109 @@ > } > > static str > -RDFexport_(str url, bat M, bat S, bat P, bat O) { > - str ret = "RDFexport failed to obtain bats"; > +RDFexport_(str url, bat M, bat S, bat P, bat O, str syntax) > +{ > + str ret = MAL_SUCCEED; > BAT *map = BATdescriptor(M); > - if (map) { > + (void)url; > + if (map) > + { > BAT *sb = BATdescriptor(S); > - if (sb) { > + if (sb) > + { > BAT *pb = BATdescriptor(P); > - if (pb) { > + if (pb) > + { > BAT *ob = BATdescriptor(O); > - if (ob) { > - lng i, cnt = BATcount(ob); > + if (ob) > + { > + raptor_serializer* rdf_serializer; > + const char *filename="/ufs/antonell/Desktop/output.rdf"; > + lng i, cnt = BATcount(ob); > oid *o = (oid *) Tloc(ob, BUNfirst(ob)); > oid *p = (oid *) Tloc(pb, BUNfirst(pb)); > oid *s = (oid *) Tloc(sb, BUNfirst(sb)); > //var_t position in heap > var_t *off = (var_t *) Tloc(map, BUNfirst(map)); > str base = (str) map->T->vheap->base;//heap first position > - > - for(i=0; i <cnt; i++) { > - str ov = base + off[o[i]]; > - str pv = base + off[p[i]]; > - str sv = base + off[s[i]]; > - stream_printf(GDKout, "'%s' '%s' '%s' '%s'\n", ov, pv, sv, url); > + > + raptor_init(); > + rdf_serializer = raptor_new_serializer(syntax); > + > + if (rdf_serializer) > + { > + char ret_array[RET_MAXLEN]; > + str ret_ptr = ret_array; > + ret_array[0] = '\0'; > + raptor_serializer_set_error_handler(rdf_serializer, &ret_ptr, message_handler); > + raptor_serializer_set_warning_handler(rdf_serializer, &ret_ptr, message_handler); > + if (raptor_serialize_start_to_filename(rdf_serializer, filename) == 0) > + { > + for(i=0; i <cnt; i++) > + { > + if (strlen(ret_ptr)==0) > + { > + raptor_statement *triple; > + str ov = base + off[o[i]]; > + str pv = base + off[p[i]]; > + str sv = base + off[s[i]]; > + //stream_printf(GDKout, "'%s' '%s' '%s' '%s'\n", ov, pv, sv, url); > + /* Make a triple with URI subject, URI predicate, literal object */ > + triple=GDKmalloc(sizeof(raptor_statement)); > + > + triple->subject=(void*)raptor_new_uri((const unsigned char*)sv); > + triple->subject_type=RAPTOR_IDENTIFIER_TYPE_RESOURCE; > + > + triple->predicate=(void*)raptor_new_uri((const unsigned char*)pv); > + triple->predicate_type=RAPTOR_IDENTIFIER_TYPE_RESOURCE; > + > + triple->object=ov; > + triple->object_type=RAPTOR_IDENTIFIER_TYPE_LITERAL; > + //triple->object_literal_language=(const unsigned char*)"en"; > + triple->object_literal_language=NULL; > + triple->object_literal_datatype = NULL; > + /* Write the triple */ > + if (raptor_serialize_statement(rdf_serializer, triple)!=0) > + ret="failed to serialize a statement"; > + > + /* Delete the triple */ > + raptor_free_uri((raptor_uri*)triple->subject); > + raptor_free_uri((raptor_uri*)triple->predicate); > + GDKfree(triple); > + } > + } > + ret = MAL_SUCCEED; > + raptor_serialize_end(rdf_serializer); > + } > + else > + ret = "raptor_serialize_start failed"; > + raptor_free_serializer(rdf_serializer); > } > - ret = MAL_SUCCEED; > + else > + ret = "raptor_new_serializer failed"; > + raptor_finish(); > BBPreleaseref(ob->batCacheid); > - } > + } > + else > + ret = "Failed to obtain O bat"; > BBPreleaseref(pb->batCacheid); > - } > + } > + else > + ret = "Failed to obtain P bat"; > BBPreleaseref(sb->batCacheid); > - } > + } > + else > + ret = "Failed to obtain S bat"; > BBPreleaseref(map->batCacheid); > - } > + } > + else > + ret = "Failed to obtain MAP bat"; > return ret; > } > > str > RDFexport(str *url, bat M, bat S, bat P, bat O) { > - str ret = RDFexport_(*url, M, S, P, O); > + str ret = RDFexport_(*url, M, S, P, O, "dummy syntax"); > if (ret != MAL_SUCCEED) > throw(MAL, "RDFexport", ret); > return MAL_SUCCEED; > @@ -513,23 +552,23 @@ > > ret = "RDFimportGraph: sorting failed"; > if (sorted) { > - ret = @:sort(sb, pb, ob, Spo, sPo, spO)@ > + ret = @:sort(sb,pb,ob,Spo,sPo,spO)@ > if (ret == MAL_SUCCEED) { > - ret = @:sort(sb, ob, pb, Sop, sOp, soP)@ > + ret = @:sort(sb,ob,pb,Sop,sOp,soP)@ > if (ret == MAL_SUCCEED) { > BBPreclaim(sorted); > sorted = BATmirror(BATsort(BATmirror(pb))); > if (sorted) { > - ret = @:sort(pb, ob,sb,Pos,pOs,poS)@ > + ret = @:sort(pb,ob,sb,Pos,pOs,poS)@ > if (ret == MAL_SUCCEED) { > - ret = @:sort(pb, sb,ob,Pso,pSo,psO)@ > + ret = @:sort(pb,sb,ob,Pso,pSo,psO)@ > if (ret == MAL_SUCCEED) { > BBPreclaim(sorted); > sorted = BATmirror(BATsort(BATmirror(ob))); > if (sorted) { > - ret = @:sort(ob, pb, sb, Ops, oPs, opS)@ > + ret = @:sort(ob,pb,sb,Ops,oPs,opS)@ > if (ret == MAL_SUCCEED) { > - ret = @:sort(ob, sb, pb,Osp,oSp,osP)@ > + ret = @:sort(ob,sb,pb,Osp,oSp,osP)@ > if (ret == MAL_SUCCEED) { > ret = MAL_SUCCEED; > goto succeed; > @@ -572,15 +611,84 @@ > throw(MAL, "RDFimportGraph", ret); > return MAL_SUCCEED; > } > -str RDFexportGraph(str *url, str *db, str *graph) { > + > +static str getBatNames(str db, str graph, char *s_buf,char *p_buf,char *o_buf, str order) > +{ > + str err = "order may be only one among \"spo\",\"sop\",\"pso\",\"pos\",\"osp\",\"ops\""; > + if (strlen(order)>3) > + return err; > + if (strcmp(order, "spo")==0) > + { > + snprintf(s_buf, 1024, "%s_%s_Spo", db, graph); > + snprintf(p_buf, 1024, "%s_%s_sPo", db, graph); > + snprintf(o_buf, 1024, "%s_%s_spO", db, graph); > + } > + else if (strcmp(order, "sop")==0) > + { > + snprintf(s_buf, 1024, "%s_%s_Sop", db, graph); > + snprintf(o_buf, 1024, "%s_%s_sOp", db, graph); > + snprintf(p_buf, 1024, "%s_%s_soP", db, graph); > + } > + else if (strcmp(order, "pos")==0) > + { > + snprintf(p_buf, 1024, "%s_%s_Pos", db, graph); > + snprintf(o_buf, 1024, "%s_%s_pOs", db, graph); > + snprintf(s_buf, 1024, "%s_%s_poS", db, graph); > + } > + else if (strcmp(order, "pso")==0) > + { > + snprintf(p_buf, 1024, "%s_%s_Pso", db, graph); > + snprintf(s_buf, 1024, "%s_%s_pSo", db, graph); > + snprintf(o_buf, 1024, "%s_%s_psO", db, graph); > + } > + else if (strcmp(order, "osp")==0) > + { > + snprintf(o_buf, 1024, "%s_%s_Osp", db, graph); > + snprintf(s_buf, 1024, "%s_%s_oSp", db, graph); > + snprintf(p_buf, 1024, "%s_%s_osP", db, graph); > + } > + else if (strcmp(order, "ops")==0) > + { > + snprintf(o_buf, 1024, "%s_%s_Ops", db, graph); > + snprintf(p_buf, 1024, "%s_%s_oPs", db, graph); > + snprintf(s_buf, 1024, "%s_%s_opS", db, graph); > + } > + else > + return err; > + return MAL_SUCCEED; > +} > + > +static str checkSyntax(str syntax) > +{ > + if ( //!strcmp(syntax, "atom") || > + !strcmp(syntax, "ntriples") || > + !strcmp(syntax, "rdfxml") || > + !strcmp(syntax, "rdfxml-abbrev") || > + !strcmp(syntax, "rdfxml-xmp") || > + !strcmp(syntax, "turtle") || > + //!strcmp(syntax, "rdfxml-xmp") || > + !strcmp(syntax, "rss-1.0") || > + !strcmp(syntax, "dot")) > + { > + return MAL_SUCCEED; > + } > + else > + return "syntax may be only one among \"atom\",\"ntriples\",\"rdfxml\",\"rdfxml-abbrev\",\"rdfxml-xmp\",\"turtle\",\"rdfxml-xmp\",\"rss-1.0\",\"dot\""; > +} > + > +str RDFexportGraph(int *d, str *url, str *db, str *graph, str *order, str *syntax) > +{ > char m_buf[1024], o_buf[1024], p_buf[1024], s_buf[1024]; > bat map, ob, pb, sb; > - str ret = "RDFexportGraph failed to obtain bats"; > - > + str ret; > + (void)d; > + ret = checkSyntax(*syntax); > + if (ret!= MAL_SUCCEED) > + throw(MAL, "RDFexportGraph", ret); > + ret = getBatNames(*db, *graph, s_buf, p_buf, o_buf, *order); > + if (ret!= MAL_SUCCEED) > + throw(MAL, "RDFexportGraph", ret); > snprintf(m_buf, 1024, "%s_%s_map", *db, *graph); > - snprintf(p_buf, 1024, "%s_%s_Ops", *db, *graph); > - snprintf(o_buf, 1024, "%s_%s_oPs", *db, *graph); > - snprintf(s_buf, 1024, "%s_%s_opS", *db, *graph); > map = BBPindex(m_buf); > if (map) { > sb = BBPindex(s_buf); > @@ -589,11 +697,19 @@ > if (pb) { > ob = BBPindex(o_buf); > if (ob) { > - ret = RDFexport_(*url, map, sb, pb, ob); > + ret = RDFexport_(*url, map, sb, pb, ob, *syntax); > } > + else > + ret = "Failed to obtain O bat"; > } > + else > + ret = "Failed to obtain P bat"; > } > + else > + ret = "Failed to obtain S bat"; > } > + else > + ret = "Failed to obtain map bat"; > if (ret != MAL_SUCCEED) > throw(MAL, "RDFexportGraph", ret); > return MAL_SUCCEED; > > > ------------------------------------------------------------------------- > Check out the new SourceForge.net Marketplace. > It's the best place to buy or sell services for > just about anything Open Source. > http://ad.doubleclick.net/clk;164216239;13503038;w?http://sf.net/marketplace > _______________________________________________ > Monetdb-checkins mailing list > Monetdb-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-checkins

2 1

[Monetdb-developers] blobs performance
by Henrik Vendelbo 30 Mar '08

30 Mar '08

I am working through the options for clustered file management. Normally I would expect fairly poor performance of binary file storage in a database. Hence I would favor using the file system, and sync across the machines in the cluster. Being quite different from common relational implementations, how well does Monet handle blobs of 1-10Mb? As a side question is there a support for multi-master replication? The technology looks really impressive, I hope to be able to base my future projects on MonetDB Thanks, Henrik

2 1

Re: [Monetdb-developers] Monetdb-developers Digest, Vol 22, Issue 4
by Peter Boncz 28 Mar '08

28 Mar '08

Hi, I read the digests of the monetdb mailing list, which in this case took more than a month to send me this discussion. That's pretty useless.. I will switch to immediate mails. Anyway, Jennie is currently working on getting the repeatable reads transactions working. This means that by passing a request-ID you will be guaranteed that all queries and updates execute on the same database image, i.e. a transaction. In case of update requests in the same transaction, these will be executed atomically using a 2PC protocol. While this functionality is designed to be triggerered from XQuery queries, you client API could also generate the appropriate messages. It may not answer you needs regarding performance (you would send a read-request, update-request, prepare-request and commit-request), but it would answer your functional requirements AFAIS. Peter > Message: 1 > Date: Wed, 19 Mar 2008 12:30:20 +0100 > From: "Keulen, M. van (Maurice)" <m.vankeulen(a)utwente.nl> > Subject: Re: [Monetdb-developers] [XQ] Mixing update/docmgmt/query > To: monetdb-developers(a)lists.sourceforge.net > Message-ID: <47E0F94C.4000709(a)utwente.nl> > Content-Type: text/plain; charset=windows-1252; format=flowed > > Ying Zhang wrote: > > > >> Jennie: my SOAP-messages do not come from MonetDB/XQuery, but from > >> outside (i.e., some scripting language). > > > > Then, an order can only be guaranteed if you, in your > scripts, send a > > message, wait for result, and send the next message. In all other > > cases, unfortunately, no guarantee, as an xrpc server immediately > > start handling a request when it's received. > > > > Jennie > > I thought so already. This is why I don't like to have to > send multiple > requests to do one thing (transaction). I'd rather be able to > send one > request to both do, say, an update and return a result, or add a > document and return a result. The system so far doesn't allow > me to do > this, because I cannot define one query (or function in a > module) that > does both because of typing restrictions, and I also cannot send two > separate queries in one request because of restrictions in the XRPC > protocol. > > Any thoughts about achieving what I want? > > Maurice. > > -- > ---------------------------------------------------------------------- > Dr.Ir. M. van Keulen - Assistant Professor, Data Management Technology > Univ. of Twente, Dept of EEMCS, POBox 217, 7500 AE Enschede, > Netherlands > Email: m.vankeulen(a)utwente.nl, Phone: +31 534893688, Fax: +31 > 534892927 > Room: ZI 3039, WWW: http://www.cs.utwente.nl/~keulen *********************************

3 8

[Monetdb-developers] MonetDB testing problems
by Stefan Manegold 27 Mar '08

27 Mar '08

Dear fellow MonetDB developers, the recently introduced archiving of testing result summaries seems to require more storage than expected --- I don't know, yet, whether this is indeed inherent in the approach, or "just" some bug --- filling up the disk space on our testing server, and thus preventing testing from running properly. I will not have any time to investigate (let alone fix) the problem this week --- in fact, given other obligations, I'm not sure whether I will find any time before May 1st ... Hence, there might be some time ahead without proper/reliable MonetDB testing. I'll keep you posted, in case/once I managed to solve the problem... Stefan -- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4312 |

2 2

[Monetdb-developers] [PF] Question about timings
by Keulen, M. van (Maurice) 25 Mar '08

25 Mar '08

Hi, A student of mine is trying to speed up some queries and is puzzled about the timing-readings he gets, for example: Trans 19681.811 msec Shred 0.000 msec Query 16384.143 msec Print 81.180 msec Timer 55087.438 msec The total of the first four 19.7+0+16.4+0.08 is approximately 36.2 seconds. "Timer" gives 55.1 seconds. Where are the remaining 18.9 seconds going? By the way, what do these measures precisely indicate? "Query", for example, can be interpreted as query compilation or query execution. Maurice. -- ---------------------------------------------------------------------- Dr.Ir. M. van Keulen - Assistant Professor, Data Management Technology Univ. of Twente, Dept of EEMCS, POBox 217, 7500 AE Enschede, Netherlands Email: m.vankeulen(a)utwente.nl, Phone: +31 534893688, Fax: +31 534892927 Room: ZI 3039, WWW: http://www.cs.utwente.nl/~keulen

2 3

[Monetdb-developers] Experimental testweb history available
by Fabian Groffen 20 Mar '08

20 Mar '08

Hi all, This is just to inform you all of a recently added "feature" (to both Stable and Current ;) ) in our beloved TestWeb. We added a "history" function to parts of TestWeb's output, which can come in handy if you're chasing pavem^Wbugs, or just interested in (since) when happened what. First off, a nice, nifty and sexy GUI is missing for sure, but who knows what the future will bring :D The MonetDB PermaStore (it is *not* powered by any MonetDB technology) http://www.ins.cwi.nl/~monet/permastore/ At this URL you will find all the output of the compilation check script, that some of you may already have been confronted with. In particular, here you can find per day the skeletons of the email messages sent out, and the annotated HTML versions of the build logs. Improvements in this area are low hanging fruit, but they just require time. Note that the aforementioned URL is still sort of experimental only. The MonetDB TestWeb 30-day archive http://monetdb.cwi.nl/testing/projects/monetdb/<branch>/<module>/.<thing>..<german_date>/ where <german_date> = yyyy.mm.dd If you know what branch (Stable, Current), module (buildtools, MonetDB, sql, ...) and thing (Makes3, Int.32.32.d-Fedora8, etc.) should be then you can inspect the TestWeb output till 30 days back in time. Note that the full 30 days are not yet generated, so only partially available.

3 2

[Monetdb-developers] [XQ] Mixing update/docmgmt/query
by Keulen, M. van (Maurice) 19 Mar '08

19 Mar '08

Hi, I know that the XQUF prohibits it, but for writing robust functions, it is often necessary to mix query and update/docmgmt functionality. Some examples: * Expression that queries/updates document, but if the document does not exist, adds an empty version of it: let $name := "foo.xml" let $emptydoc := <root/> let $doc := if (empty(pf:documents()[.=$name])) then (put($emptydoc,$name),$emptydoc) else doc("foo.xml") return $doc//bar * Expression that adds to a log-file: let $log := doc("log.xml") let $expr := ...some expression... return (do insert <entry>...{$expr}...<entry> as last into $log,$expr) Each of these patterns gives you a "!type error: err:XUST0101: illegal mix of updating and non-updating expressions" error. Isn't is possible to somehow allow such expressions? It is against the standard, so it probably requires some server configuration parameter, but it would be really really useful to have it. Mixing queries with update/docmgmt is known to cause many semantical problems, but only if these updates are visible within the query. Looking at the implementation based on pending update lists, then each of these patterns is harmless. If you declare that no updates are visible within the transaction (which is against the standard), then allowing these queries is I guess not hard: it simply means that with updating expressions and document management expressions you do return a result. Another thing I've been thinking about in this context: If I send two SOAP-messages to a server on its XRPC-port, am I guaranteed that they are served in the same order? I could perhaps, with a lot of duplication of code, separate update/docmgmt from a query, i.e., implement these things with two requests to the server. I really dislike such a solution, though. Looking forward to comments :-) Maurice. -- ---------------------------------------------------------------------- Dr.Ir. M. van Keulen - Assistant Professor, Data Management Technology Univ. of Twente, Dept of EEMCS, POBox 217, 7500 AE Enschede, Netherlands Email: m.vankeulen(a)utwente.nl, Phone: +31 534893688, Fax: +31 534892927 Room: ZI 3039, WWW: http://www.cs.utwente.nl/~keulen

3 5

[Monetdb-developers] Bug in Python interface converters.py Feb2008-SuperBall
by Andrei Tovtchigretchko 15 Mar '08

15 Mar '08

MonetSQLdb.converters.escape(x, conv=None) should return 'tuple' for any input sequence type (currently it returns tuple only if it receives tuple, and returns list if it receives list). The return result is used as a second argument in expression 'query = query % self.connection.literal(args)' in cursors.py. The '%' operator raises an exception on anything but tuple. Also, when 'escape()' cannot convert some argument, it should immediately raise a descriptive exception. Currently it silently drops the argument from the result sequence, which causes an exception to be raised later in the calling code. That exception reads like this: """ProgrammingError: not enough arguments for format string""" This unnecessarily complicates debugging. Thank you, Andrey _________________________________________________________________ Watch “Cause Effect,” a show about real people making a real difference. Learn more. http://im.live.com/Messenger/IM/MTV/?source=text_watchcause

2 1

Re: [Monetdb-developers] [Monetdb-checkins] MonetDB5/src/mal mal_recycle.mx, , 1.49, 1.50
by Stefan Manegold 15 Mar '08

15 Mar '08

On Sat, Mar 15, 2008 at 08:41:35AM +0000, Martin Kersten wrote: > Update of /cvsroot/monetdb/MonetDB5/src/mal > In directory sc8-pr-cvs16.sourceforge.net:/tmp/cvs-serv17899 > > Modified Files: > mal_recycle.mx > Log Message: > Silence Windows compiler FYI: `man bzero` says: ======== CONFORMING TO 4.3BSD. This function is deprecated (marked as LEGACY in POSIX.1-2001): use memset(3) in new programs. ======== Stefan > Index: mal_recycle.mx > =================================================================== > RCS file: /cvsroot/monetdb/MonetDB5/src/mal/mal_recycle.mx,v > retrieving revision 1.49 > retrieving revision 1.50 > diff -u -d -r1.49 -r1.50 > --- mal_recycle.mx 14 Mar 2008 09:43:07 -0000 1.49 > +++ mal_recycle.mx 15 Mar 2008 08:41:33 -0000 1.50 > @@ -260,7 +260,7 @@ > newMalBlkStmt(recycleBlk,recycleBlk->ssize); > > used= alloca(recycleBlk->vtop); > - bzero(used, recycleBlk->vtop); > + memset(used,0, recycleBlk->vtop); > > /* set all used variables */ > for (i = 0; i < pc; i++){ > > > ------------------------------------------------------------------------- > This SF.net email is sponsored by: Microsoft > Defy all challenges. Microsoft(R) Visual Studio 2008. > http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/ > _______________________________________________ > Monetdb-checkins mailing list > Monetdb-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-checkins -- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4312 |

1 0

Re: [Monetdb-developers] [Monetdb-checkins] MonetDB5/src/modules/mal recycle.mx, , 1.9, 1.10
by Stefan Manegold 10 Mar '08

10 Mar '08

On Mon, Mar 10, 2008 at 11:21:59AM +0000, Romulo Goncalves wrote: > Update of /cvsroot/monetdb/MonetDB5/src/modules/mal > In directory sc8-pr-cvs16.sourceforge.net:/tmp/cvs-serv24521 > > Modified Files: > recycle.mx > Log Message: > Check in the MALblk exist to avoid a mserver crash when we to > recycle.dump() before any query execution why do you check the loop-invariant condition with every iteration of the loop? Stefan > Index: recycle.mx > =================================================================== > RCS file: /cvsroot/monetdb/MonetDB5/src/modules/mal/recycle.mx,v > retrieving revision 1.9 > retrieving revision 1.10 > diff -u -d -r1.9 -r1.10 > --- recycle.mx 9 Mar 2008 15:47:30 -0000 1.9 > +++ recycle.mx 10 Mar 2008 11:21:47 -0000 1.10 > @@ -147,7 +147,7 @@ > stream_printf(GDKout,"Recycler catalog \n"); > > /* and dump the statistics */ > - for(i=0; i< recycleBlk->stop; i++){ > + for(i=0; recycleBlk && i< recycleBlk->stop; i++){ > stream_printf(GDKout,"#%4d count=%3d ticks=%4d %s\n", i, > recycleBlk->profiler[i].counter, > recycleBlk->profiler[i].ticks, > > > ------------------------------------------------------------------------- > This SF.net email is sponsored by: Microsoft > Defy all challenges. Microsoft(R) Visual Studio 2008. > http://clk.atdmt.com/MRT/go/vse0120000070mrt/direct/01/ > _______________________________________________ > Monetdb-checkins mailing list > Monetdb-checkins(a)lists.sourceforge.net > https://lists.sourceforge.net/lists/listinfo/monetdb-checkins > -- | Dr. Stefan Manegold | mailto:Stefan.Manegold@cwi.nl | | CWI, P.O.Box 94079 | http://www.cwi.nl/~manegold/ | | 1090 GB Amsterdam | Tel.: +31 (20) 592-4212 | | The Netherlands | Fax : +31 (20) 592-4312 |

2 1