- developers-list - monetdb.org

Re: MonetDB: msk-type - Initial support for the msk type.
by Martin Kersten 24 Apr '20

24 Apr '20

wil je dat ik het doortrek naar MAL niveau? > On 24 Apr 2020, at 17:30, Sjoerd Mullender <commits+sjoerd=acm.org(a)monetdb.org> wrote: > > Changeset: abf00b42a524 for MonetDB > URL: https://dev.monetdb.org/hg/MonetDB?cmd=changeset;node=abf00b42a524 > Modified Files: > clients/Tests/exports.stable.out > gdk/gdk.h > gdk/gdk_atoms.c > gdk/gdk_bat.c > gdk/gdk_batop.c > gdk/gdk_calc.c > gdk/gdk_hash.c > gdk/gdk_value.c > monetdb5/mal/mal_type.c > monetdb5/modules/atoms/inet.c > monetdb5/modules/kernel/bat5.c > sql/backends/monet5/UDF/capi/capi.c > sql/backends/monet5/sql_result.c > Branch: msk-type > Log Message: > > Initial support for the msk type. > > > diffs (truncated from 1402 to 300 lines): > > diff --git a/clients/Tests/exports.stable.out b/clients/Tests/exports.stable.out > --- a/clients/Tests/exports.stable.out > +++ b/clients/Tests/exports.stable.out > @@ -1273,7 +1273,7 @@ str INETnetmask(inet *retval, const inet > str INETnetwork(inet *retval, const inet *val); > str INETnew(inet *retval, str *in); > const inet *INETnull(void); > -str INETsetmasklen(inet *retval, const inet *val, const int *msk); > +str INETsetmasklen(inet *retval, const inet *val, const int *mask); > str INETtext(str *retval, const inet *val); > ssize_t INETtoString(str *retval, size_t *len, const inet *handle, bool external); > str INSPECTatom_names(bat *ret); > diff --git a/gdk/gdk.h b/gdk/gdk.h > --- a/gdk/gdk.h > +++ b/gdk/gdk.h > @@ -425,7 +425,8 @@ > > enum { > TYPE_void = 0, > - TYPE_bit, > + TYPE_msk, /* bit mask */ > + TYPE_bit, /* TRUE, FALSE, or nil */ > TYPE_bte, > TYPE_sht, > TYPE_bat, /* BAT id: index in BBPcache */ > @@ -445,6 +446,7 @@ enum { > TYPE_any = 255, /* limit types to <255! */ > }; > > +typedef bool msk; > typedef int8_t bit; > typedef int8_t bte; > typedef int16_t sht; > @@ -602,6 +604,7 @@ typedef struct { > oid oval; > sht shval; > bte btval; > + msk mval; > flt fval; > ptr pval; > bat bval; > @@ -626,7 +629,7 @@ gdk_export void VALclear(ValPtr v); > gdk_export ValPtr VALset(ValPtr v, int t, void *p); > gdk_export void *VALget(ValPtr v); > gdk_export int VALcmp(const ValRecord *p, const ValRecord *q); > -gdk_export int VALisnil(const ValRecord *v); > +gdk_export bool VALisnil(const ValRecord *v); > > /* > * @- The BAT record > @@ -746,7 +749,10 @@ typedef struct BAT { > > typedef struct BATiter { > BAT *b; > - oid tvid; > + union { > + oid tvid; > + bool tmsk; > + }; > } BATiter; > > /* macros to hide complexity of the BAT structure */ > @@ -772,6 +778,33 @@ typedef struct BATiter { > #define tprops T.props > > > +/* some access functions for the bitmask type */ > +static inline void > +mskSet(BAT *b, BUN p) > +{ > + ((uint32_t *) b->theap.base)[p / 32] |= 1U << (p % 32); > +} > + > +static inline void > +mskClr(BAT *b, BUN p) > +{ > + ((uint32_t *) b->theap.base)[p / 32] &= ~(1U << (p % 32)); > +} > + > +static inline void > +mskSetVal(BAT *b, BUN p, msk v) > +{ > + if (v) > + mskSet(b, p); > + else > + mskClr(b, p); > +} > + > +static inline msk > +mskGet(BAT *b, BUN p) > +{ > + return ((uint32_t *) b->theap.base)[p / 32] & (1U << (p % 32)); > +} > > /* > * @- Heap Management > @@ -925,7 +958,11 @@ gdk_export BUN BUNfnd(BAT *b, const void > > #define Tsize(b) ((b)->twidth) > > -#define tailsize(b,p) ((b)->ttype?((size_t)(p))<<(b)->tshift:0) > +#define tailsize(b,p) ((b)->ttype ? \ > + (ATOMstorage((b)->ttype) == TYPE_msk ? \ > + (((size_t) (p) + 31) / 32) * 4 : \ > + ((size_t) (p)) << (b)->tshift) : \ > + 0) > > #define Tloc(b,p) ((void *)((b)->theap.base+(((size_t)(p))<<(b)->tshift))) > > @@ -935,7 +972,7 @@ typedef var_t stridx_t; > > #define BUNtvaroff(bi,p) VarHeapVal((bi).b->theap.base, (p), (bi).b->twidth) > > -#define BUNtloc(bi,p) Tloc((bi).b,p) > +#define BUNtloc(bi,p) (ATOMstorage((bi).b->ttype) == TYPE_msk ? Tmsk(&(bi), p) : Tloc((bi).b,p)) > #define BUNtpos(bi,p) Tpos(&(bi),p) > #define BUNtvar(bi,p) (assert((bi).b->ttype && (bi).b->tvarsized), (void *) (Tbase((bi).b)+BUNtvaroff(bi,p))) > #define BUNtail(bi,p) ((bi).b->ttype?(bi).b->tvarsized?BUNtvar(bi,p):BUNtloc(bi,p):BUNtpos(bi,p)) > @@ -1519,6 +1556,8 @@ Tputvalue(BAT *b, BUN p, const void *v, > break; > #endif > } > + } else if (b->ttype == TYPE_msk) { > + mskSetVal(b, p, * (msk *) v); > } else { > return ATOMputFIX(b->ttype, Tloc(b, p), v); > } > @@ -1528,7 +1567,11 @@ Tputvalue(BAT *b, BUN p, const void *v, > static inline gdk_return __attribute__((__warn_unused_result__)) > tfastins_nocheck(BAT *b, BUN p, const void *v, int s) > { > - b->theap.free += s; > + if (ATOMstorage(b->ttype) == TYPE_msk) { > + if (p % 32 == 0) > + b->theap.free += 4; > + } else > + b->theap.free += s; > return Tputvalue(b, p, v, false); > } > > @@ -1676,6 +1719,7 @@ VALptr(const ValRecord *v) > { > switch (ATOMstorage(v->vtype)) { > case TYPE_void: return (const void *) &v->val.oval; > + case TYPE_msk: return (const void *) &v->val.mval; > case TYPE_bte: return (const void *) &v->val.btval; > case TYPE_sht: return (const void *) &v->val.shval; > case TYPE_int: return (const void *) &v->val.ival; > @@ -1734,8 +1778,6 @@ gdk_export void *THRdata[THREADDATA]; > #define THRget_errbuf(t) ((char*)t->data[2]) > #define THRset_errbuf(t,b) (t->data[2] = b) > > -#ifndef GDK_NOLINK > - > static inline bat > BBPcheck(bat x, const char *y) > { > @@ -1773,7 +1815,12 @@ Tpos(BATiter *bi, BUN p) > return (void*)&bi->tvid; > } > > -#endif > +static inline void * > +Tmsk(BATiter *bi, BUN p) > +{ > + bi->tmsk = mskGet(bi->b, p); > + return &bi->tmsk; > +} > > /* > * @+ Transaction Management > diff --git a/gdk/gdk_atoms.c b/gdk/gdk_atoms.c > --- a/gdk/gdk_atoms.c > +++ b/gdk/gdk_atoms.c > @@ -36,6 +36,12 @@ > * NaN). */ > > static int > +mskCmp(const msk *l, const msk *r) > +{ > + return (*l > *r) - (*l < *r); > +} > + > +static int > bteCmp(const bte *l, const bte *r) > { > return (*l > *r) - (*l < *r); > @@ -450,6 +456,42 @@ voidWrite(const void *a, stream *s, size > * experiments showed that library function is even slightly faster and we > * now also support True/False (and trUe/FAlSE should this become a thing). > */ > +static ssize_t > +mskFromStr(const char *src, size_t *len, msk **dst, bool external) > +{ > + const char *p = src; > + > + (void) external; > + atommem(sizeof(msk)); > + > + if (strNil(src)) > + return -1; > + > + while (GDKisspace(*p)) > + p++; > + if (*p == '0') { > + **dst = 0; > + p++; > + } else if (*p == '1') { > + **dst = 1; > + p++; > + } else { > + return -1; > + } > + while (GDKisspace(*p)) > + p++; > + return (ssize_t) (p - src); > +} > + > +static ssize_t > +mskToStr(char **dst, size_t *len, const msk *src, bool external) > +{ > + (void) external; > + atommem(2); > + strcpy(*dst, *src ? "1" : "0"); > + return 1; > +} > + > ssize_t > bitFromStr(const char *src, size_t *len, bit **dst, bool external) > { > @@ -1202,6 +1244,17 @@ atomDesc BATatoms[MAXATOMS] = { > .atomCmp = (int (*)(const void *, const void *)) bteCmp, > .atomHash = (BUN (*)(const void *)) bteHash, > }, > + [TYPE_msk] = { > + .name = "msk", > + .storage = TYPE_msk, > + .linear = false, > + .size = 1, /* really 1/8 */ > + .atomFromStr = (ssize_t (*)(const char *, size_t *, void **, bool)) mskFromStr, > + .atomToStr = (ssize_t (*)(char **, size_t *, const void *, bool)) mskToStr, > +// .atomRead = (void *(*)(void *, stream *, size_t)) mskRead, > +// .atomWrite = (gdk_return (*)(const void *, stream *, size_t)) mskWrite, > + .atomCmp = (int (*)(const void *, const void *)) mskCmp, > + }, > [TYPE_bte] = { > .name = "bte", > .storage = TYPE_bte, > diff --git a/gdk/gdk_bat.c b/gdk/gdk_bat.c > --- a/gdk/gdk_bat.c > +++ b/gdk/gdk_bat.c > @@ -184,7 +184,9 @@ COLnew(oid hseq, int tt, BUN cap, role_t > /* round up to multiple of BATTINY */ > if (cap < BUN_MAX - BATTINY) > cap = (cap + BATTINY - 1) & ~(BATTINY - 1); > - if (cap < BATTINY) > + if (ATOMstorage(tt) == TYPE_msk && cap < 8*BATTINY) > + cap = 8*BATTINY; > + else if (cap < BATTINY) > cap = BATTINY; > /* limit the size */ > if (cap > BUN_MAX) > @@ -197,6 +199,9 @@ COLnew(oid hseq, int tt, BUN cap, role_t > BATsetdims(bn); > bn->batCapacity = cap; > > + if (ATOMstorage(tt) == TYPE_msk) > + cap /= 8; /* 8 values per byte */ > + > /* alloc the main heaps */ > if (tt && HEAPalloc(&bn->theap, cap, bn->twidth) != GDK_SUCCEED) { > goto bailout; > @@ -339,12 +344,16 @@ BATattach(int tt, const char *heapfile, > GDKerror("heapfile size not integral number of atoms\n"); > return NULL; > } > - if ((size_t) (st.st_size / atomsize) > (size_t) BUN_MAX) { > + if (ATOMstorage(tt) == TYPE_msk ? > + (st.st_size > (off_t) (BUN_MAX / 8)) : > + ((size_t) (st.st_size / atomsize) > (size_t) BUN_MAX)) { > fclose(f); > GDKerror("heapfile too large\n"); > return NULL; > } > - cap = (BUN) (st.st_size / atomsize); > + cap = (BUN) (ATOMstorage(tt) == TYPE_msk ? > + st.st_size * 8 : > + st.st_size / atomsize); > bn = COLnew(0, tt, cap, role); > if (bn == NULL) { > fclose(f); > @@ -371,8 +380,8 @@ BATattach(int tt, const char *heapfile, > bn->trevsorted = false; > bn->tkey = false; > } else { > - bn->tsorted = true; > - bn->trevsorted = true; > + bn->tsorted = ATOMlinear(tt); > + bn->trevsorted = ATOMlinear(tt); > bn->tkey = true; > _______________________________________________ > checkin-list mailing list > checkin-list(a)monetdb.org > https://www.monetdb.org/mailman/listinfo/checkin-list

1 0

New Jun2020 stable branch created
by Sjoerd Mullender 18 Mar '20

18 Mar '20

I have created a new stable branch for the next feature release. This branch was created in order to stabilize the code before the feature release. >From now on *only* bug fixes may go to this new stable branch. I reserve the right to undo any changes that do not comply with this rule. I also reserve the right to admit changes that are not strictly bug fixes. The name for the branch is Jun2020. So in each of your stable checkout directories you can do hg pull hg update -rJun2020 to update to this new branch. -- Sjoerd Mullender

1 0

Announcement: New Nov2019-SP2 Bugfix release of MonetDB suite
by Sjoerd Mullender 21 Feb '20

21 Feb '20

The MonetDB team at MonetDB BV is pleased to announce the Nov2019-SP2 bugfix release of the MonetDB suite of programs. More information about MonetDB can be found on our website at <https://www.monetdb.org/>. For details on this release, please see the release notes at <https://www.monetdb.org/Downloads/ReleaseNotes>. As usual, the download location is <https://www.monetdb.org/downloads/>. Nov 2019-SP2 bugfix release (11.35.15) Bug Fixes * 6805: Using the cascade operator in a drop table statement ends in an exit from the Monetdb shell. * 6807: Median_avg and quantile_avg ignore NULL values * 6815: query with ifthenelse() crashes mserver5 * 6816: Monetdb Crashes on INSERT statement after ALTER statement in another connection * 6817: running analyze on a schema which contains a stream table stops with an error * 6819: functions do not persist

1 0

Re: MonetDB: gdk_tracer - small compilation fixes for windows
by Martin Kersten 27 Dec '19

27 Dec '19

Niels, A big check-in is coming. I went over the wlcr code and 'removed' the background thread. Now only rolling forward using wlr.replicate(limit) or (exclusively) the background thread using a beat. Test wlr70 still fails, because the code in wlr.c line 520 is still wrong (call wlr.replicate(now())) strftime(timelimit, sizeof(timelimit), "%Y-%m-%d %H:%M:%S.000", (void*) getArgReference(stk, pci, 1)); advice?

1 0

Announcement: New Nov2019-SP1 Bugfix release of MonetDB suite
by Sjoerd Mullender 20 Dec '19

20 Dec '19

The MonetDB team at CWI/MonetDB BV is pleased to announce the Nov2019-SP1 bugfix release of the MonetDB suite of programs. More information about MonetDB can be found on our website at <https://www.monetdb.org/>. For details on this release, please see the release notes at <https://www.monetdb.org/Downloads/ReleaseNotes>. As usual, the download location is <https://dev.monetdb.org/downloads/>. Nov 2019-SP1 bugfix release (11.35.9) Bug Fixes * 6723: columns aliases duplicates should not be allowed. automatic aliasing required. * 6724: Prepare confuses types when more than one argument is used * 6726: Python aggregation does not create aggr_group when aggregating over all rows * 6765: GRANT SELECT privilege on a subset of table columns results in access denied error when selecting the same columns from the table * 6790: Count distinct giving wrong results * 6791: str_to_time('11:40', '%H:%M') creates wrong time value * 6792: JSON path compiler accepts invalid input * 6793: cast(interval second value to int or decimal) is wrong (by a factor of 1000), cast(interval month value to decimal or floating point) fails * 6794: external name fits.listdir not bound (sys.listdir) Fatal error during initialization: * 6796: Incorrect crash time reported by monetdb tool after crash of mserver5 * 6798: json.text off by one error * 6804: DNS resolution of 0.0.0.0 fails on recent Ubuntus

1 0

Announcement: New Nov2019 Feature release of MonetDB suite
by Sjoerd Mullender 28 Nov '19

28 Nov '19

The MonetDB team at CWI/MonetDB BV is pleased to announce the Nov2019 feature release of the MonetDB suite of programs. More information about MonetDB can be found on our website at <https://www.monetdb.org/>. For details on this release, please see the release notes at <https://www.monetdb.org/Downloads/ReleaseNotes>. As usual, the download location is <https://dev.monetdb.org/downloads/>. Nov 2019 feature release (11.35.3) MonetDB5 Server * The server "console" has been removed, as has the --daemon option. The server now doesn't read from standard input anymore. The way to stop a server is by sending it a TERM signal (on Linux/Unix) or by sending it an interrupt signal (usually control-C -- on all systems). * Implemented a function bat.diffcand to calculate difference of two candidate lists. * The mtime module was completely rewritten, the atom types date, daytime, and timestamp were changed. Upgrade code for BATs containing these types has been implemented. The old daytime type used a 32 bit integer to record milliseconds since the start of the day. The new daytime type uses a 64 bit integer to record microseconds since the start of the day. The old date type recorded days since or before the year 1. The new daytime type records the day of the month and the number of months since the year -4712 separately in a single 32 bit integer of which only 26 bits are used. Dates now use the proleptic Gregorian calendar, meaning the normal Gregorian callendar has been extended backward, and before the year 1, we have the year 0. Both the old and new timestamp types are a combination of a daytime and a date value, but since those types have changed, the timestamp type has also changed. The new date type has a smaller range than the old. The new date range is from the year -4712 to the year 170049. During conversion of date and timestamp columns, the dates are clamped to this range. * The tzone and rule atom types have been removed. They were not used by any code, and they were defined in a non-portable way. * Added "mapi_ipv6" property to monet_options to force ipv6 address binding only. This property is inherited while forking from monetdbd if it is also set there. * Removed (bat)calc.between_symmetric and changed (bat)calc.between by adding a number of extra arguments, all of type :bit: symmetric, low inclusive, high inclusive, nils false. Merovingian * Added "vmmaxsize" and "memmaxsize" mserver5 options to the daemon in order to set mserver5's maximum virtual and committed memory respectively. * Added ipv6 property to monetdbd properties to force IPv6 addresses binding only. By default this property is false to allow IPv4 addresses as well. MonetDB Common * BATcalcbetween and all its variants now have an extra bool parameter "anti" to invert the test. * All forms of BATcalcbetween and VARcalcbetween have extra bool arguments for low inclusive, high inclusive and nils false. The latter causes the result to be false instead of nil if the value being checked is nil. SQL Frontend * Removed functions json.text(string) returns string and json.text(int) returns string. Their MAL implementation didn't exist, so they were meaningless. * There are new aggregate functions sys.median_avg and sys.quantile_avg that return the interpolated value if the median/quantile doesn't fall exactly on a particular row. These functions always return a value of type DOUBLE and only work for numeric types (various width integers and floating point). * Added sys.deltas ("schema" string, "table" string, "column" string) returns table ("values" bigint) system function which returns a single column with 6 values: a flag indicating if the column's upper table is cleared or not, the count of the RDONLY, RD_INS and RD_UPD_ID deltas of the column itself, the number of deleted values of the column's table, as well as the level of the current transaction in the transaction level tree. It should be used for debugging purposes only. * Added "VALUES row_list" statement as a top SQL projection statement. * The implementation of in-expression now follows a join-based approach instead of using iterative union/selects. This greatly improves performance for large in-value-lists. Furthermore the old approach has large in-value-lists generate large MAL plans. This is now no longer the case. * Strings are now limited to 1GB, double-quoted tokens are limited to 2kB. These sizes are bytes of (UTF-8 encoded) input data. Bug Fixes * 3533: SQL aggregate functions avg(), sum() and median() return an error when used on a column with datatype interval second * 6134: Query produces error: HEAPalloc: Insufficient space for HEAP of 1168033427456 bytes. * 6613: LATERAL crash /.../rel_bin.c:1473: rel2bin_table: Assertion `0' failed. * 6683: Bug in subselect * 6686: Bug in subselect (count function) * 6688: Bug in subselect (or condition) * 6689: Trying to improve the performance of SQL queries with a large list of members in IN clause. * 6695: timestamp transformation * 6700: Monetdb Bugs in Subselect statements: * 6722: window functions issues * 6740: while upgrading the database from verison (MonetDB-11.27.13) to (MonetDB-11.33.3) we are unable to bring up the database * 6754: in mclient a strang msg is reported after issueing command: set schema sys; * 6755: Assertion failure in rel_bin.c * 6756: Error in optimizer garbageCollector on merge tables select * 6757: Double free or corruption (out) * 6758: SIGSEGV in __strcmp_sse2_unaligned() * 6759: Python JSON loader creates invalid data type for strings * 6761: Error: Program contains errors.:(NONE).multiplex * 6762: mserver5 crashes on (re-)start * 6764: mserver5 crashes with corruption, double free, invalid size or invalid pointer * 6766: Missing bulk implementation for get_value and next_value calls * 6769: ProfilerStart is not threadsafe * 6771: R-devel * 6772: TRACE includes information from all active sessions * 6773: json.filter returns corrupted string when selecting JSON null value * 6774: PROD aggregation gives wrong result * 6775: NOT IN with an AND containing an OR gives wrong result * 6776: Creating a table with a full outer join query gives type with wrong digits on the joined key. * 6779: Using Windows Messages translation for errno error codes. * 6780: Wrong value of the rank function * 6781: Insert after index creation crash * 6783: AVG changes scale of its results * 6784: function sys.isauuid(string) should return false if string value cannot be converted to a UUID

1 0

Re: MonetDB: default - Compilation fixes
by Martin Kersten 08 Nov '19

08 Nov '19

On 08/11/2019 09:48, Pedro Ferreira wrote: > + ATOMIC_TYPE workers; /* Actual number of concurrent workers */ This is strange!! They were removed and turned into normal variables. Now I wonder if the correct merge has taken place. I will check the code

1 0

JOIN using OR and winerror translation
by Daniel Zvinca 22 Oct '19

22 Oct '19

Hello, During a complex join between two tables that involve an OR operation, I received the following error message: !OS: The printer is out of paper. This corresponds to error number 28 in Windows OS, yet in errno.h, 28 is for ENOSPC, which is the expected one. In the meanwhile, I fixed that join, replacing the complex condition that included one OR into a UNION ALL (conditions made that possible), but I think it's worth investigating possible optimization that would do this internally (transforming join conditions using OR with plain UNION). Anyway, if above suggestion is just something to think about (as long as it always exists a solution in the hands of the SQL developer), the winerror translation certainly requires a review. (mutils.c / int winerror(int e)); Thank you, Dan

2 1

R-devel
by Arjen P. de Vries 09 Oct '19

09 Oct '19

Hi, Just installing from source for the first time since quite a while :-) On a FC30 system, using: ./configure --disable-debug --disable-developer --disable-assert --enable-optimize the bootstrap / configure steps did not complain about R-devel missing, but then issuing make failed on not finding the Rembedded.h include. Maybe the configure script needs a patch! All fine after installing R-devel, thanks for great docs! Cheers, Arjen -- ==================================================================== ICIS, office M1.02.07 Radboud University Mercator 1 Faculty of Science Toernooiveld 212 arjen(a)cs.ru.nl NL-6525 EC Nijmegen, The Netherlands +31-(0)24-365 2354 ===================== http://www.informagus.nl/ ====================

2 1

case performance
by Daniel Zvinca 08 Oct '19

08 Oct '19

Hello, I am exploring MonetDB queries performance that are generated based on user parameters/preferences. To be more specific the where clause is generated based on a user interface mapping user input parameters straight into SQL statements. See below one example of this transformation. FIELD contains PATTERN (case sensitive/insensitive) is transformed into select ... from TABLE where (case when (user_case_sensitive_pref=1) then ("FIELD" like '%PATTERN%') else ("FIELD" ilike '%PATTERN%') end); Performance of this in comparison with query generated from within the code: char * query; if ( user_case_sensitive_pref ) query = "select ... from TABLE where \"FIELD\" like '%PATTERN%'"; else query = "select ... from TABLE where \"FIELD\" ilike '%PATTERN%'"; query_execute( query); is huge (second approach takes 0.12 sec, vs 6 sec for first one). But it requires a far more complex implementation. First I thought this has to do with user variables but even if the statement is select ... from TABLE where (case when TRUE then ("FIELD" like '%PATTERN%') else ("FIELD" ilike '%PATTERN%') end); timing is similarly slow. This narrows down to "case" performance issue, as far as I can see. Am I wrong? Thank you, Dan

3 8