Skip to content

Commit da29fc6

Browse files
author
Nikita Glukhov
committed
Add logically chunked arrays to jsonb_toaster
1 parent fdf472a commit da29fc6

File tree

5 files changed

+343
-3
lines changed

5 files changed

+343
-3
lines changed

‎contrib/jsonb_toaster/Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ MODULE_big = jsonb_toaster
44
OBJS = \
55
$(WIN32RES) \
66
jsonb_toaster.o \
7-
jsonb_toast_internals.o
7+
jsonb_toast_internals.o \
8+
jsonb_toast_array.o
89

910
EXTENSION = jsonb_toaster
1011
DATA = jsonb_toaster--1.0.sql

‎contrib/jsonb_toaster/expected/jsonb_toaster.out

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -345,3 +345,259 @@ select id, js->'c'->0->>'a' from test_jsonbz_arr order by id;
345345
19 | eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee
346346
(20 rows)
347347

348+
create table test_jsonxa_arr (id int, js jsonb toaster jsonb_toaster);
349+
insert into test_jsonxa_arr
350+
select i, (select jsonb_agg(j) from generate_series(1, (2 ^ i)::int) j)
351+
from generate_series(7, 20) i;
352+
select id, pg_column_size(js) from test_jsonxa_arr order by id;
353+
id | pg_column_size
354+
----+----------------
355+
7 | 1544
356+
8 | 894
357+
9 | 1760
358+
10 | 108
359+
11 | 168
360+
12 | 288
361+
13 | 538
362+
14 | 1168
363+
15 | 111051
364+
16 | 222124
365+
17 | 444295
366+
18 | 888615
367+
19 | 1777262
368+
20 | 3554747
369+
(14 rows)
370+
371+
select id, (select count(*) from jsonb_array_elements(js)) from test_jsonxa_arr order by id;
372+
id | count
373+
----+---------
374+
7 | 128
375+
8 | 256
376+
9 | 512
377+
10 | 1024
378+
11 | 2048
379+
12 | 4096
380+
13 | 8192
381+
14 | 16384
382+
15 | 32768
383+
16 | 65536
384+
17 | 131072
385+
18 | 262144
386+
19 | 524288
387+
20 | 1048576
388+
(14 rows)
389+
390+
select id, js -> 100 from test_jsonxa_arr order by id;
391+
id | ?column?
392+
----+----------
393+
7 | 101
394+
8 | 101
395+
9 | 101
396+
10 | 101
397+
11 | 101
398+
12 | 101
399+
13 | 101
400+
14 | 101
401+
15 | 101
402+
16 | 101
403+
17 | 101
404+
18 | 101
405+
19 | 101
406+
20 | 101
407+
(14 rows)
408+
409+
select id, js -> 200 from test_jsonxa_arr order by id;
410+
id | ?column?
411+
----+----------
412+
7 |
413+
8 | 201
414+
9 | 201
415+
10 | 201
416+
11 | 201
417+
12 | 201
418+
13 | 201
419+
14 | 201
420+
15 | 201
421+
16 | 201
422+
17 | 201
423+
18 | 201
424+
19 | 201
425+
20 | 201
426+
(14 rows)
427+
428+
select id, js -> (jsonb_array_length(js) - 1) from test_jsonxa_arr order by id;
429+
id | ?column?
430+
----+----------
431+
7 | 128
432+
8 | 256
433+
9 | 512
434+
10 | 1024
435+
11 | 2048
436+
12 | 4096
437+
13 | 8192
438+
14 | 16384
439+
15 | 32768
440+
16 | 65536
441+
17 | 131072
442+
18 | 262144
443+
19 | 524288
444+
20 | 1048576
445+
(14 rows)
446+
447+
select id, json_query(js, '$[0 to 10]' with wrapper) from test_jsonxa_arr order by id;
448+
id | json_query
449+
----+-------------------------------------
450+
7 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
451+
8 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
452+
9 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
453+
10 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
454+
11 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
455+
12 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
456+
13 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
457+
14 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
458+
15 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
459+
16 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
460+
17 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
461+
18 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
462+
19 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
463+
20 | [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]
464+
(14 rows)
465+
466+
update test_jsonxa_arr set js = jsonb_set(js, '{0}', '0');
467+
select id, (select count(*) from jsonb_array_elements(js)) from test_jsonxa_arr order by id;
468+
id | count
469+
----+---------
470+
7 | 128
471+
8 | 256
472+
9 | 512
473+
10 | 1024
474+
11 | 2048
475+
12 | 4096
476+
13 | 8192
477+
14 | 16384
478+
15 | 32768
479+
16 | 65536
480+
17 | 131072
481+
18 | 262144
482+
19 | 524288
483+
20 | 1048576
484+
(14 rows)
485+
486+
select id, js -> 0 from test_jsonxa_arr order by id;
487+
id | ?column?
488+
----+----------
489+
7 | 0
490+
8 | 0
491+
9 | 0
492+
10 | 0
493+
11 | 0
494+
12 | 0
495+
13 | 0
496+
14 | 0
497+
15 | 0
498+
16 | 0
499+
17 | 0
500+
18 | 0
501+
19 | 0
502+
20 | 0
503+
(14 rows)
504+
505+
select id, js -> 100 from test_jsonxa_arr order by id;
506+
id | ?column?
507+
----+----------
508+
7 | 101
509+
8 | 101
510+
9 | 101
511+
10 | 101
512+
11 | 101
513+
12 | 101
514+
13 | 101
515+
14 | 101
516+
15 | 101
517+
16 | 101
518+
17 | 101
519+
18 | 101
520+
19 | 101
521+
20 | 101
522+
(14 rows)
523+
524+
update test_jsonxa_arr set js = json_modify(js, set '$[0 to 3]' = '0');
525+
select id, json_query(js, '$[0 to 10]' with wrapper) from test_jsonxa_arr order by id;
526+
id | json_query
527+
----+---------------------------------------------
528+
7 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
529+
8 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
530+
9 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
531+
10 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
532+
11 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
533+
12 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
534+
13 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
535+
14 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
536+
15 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
537+
16 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
538+
17 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
539+
18 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
540+
19 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
541+
20 | ["0", "0", "0", "0", 5, 6, 7, 8, 9, 10, 11]
542+
(14 rows)
543+
544+
update test_jsonxa_arr set js = json_modify(js, insert '$[200 to 203]' = '0');
545+
select id, json_query(js, '$[200 to 210]' with wrapper) from test_jsonxa_arr order by id;
546+
id | json_query
547+
----+---------------------------------------------------------
548+
7 | ["0", "0", "0", "0"]
549+
8 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
550+
9 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
551+
10 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
552+
11 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
553+
12 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
554+
13 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
555+
14 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
556+
15 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
557+
16 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
558+
17 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
559+
18 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
560+
19 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
561+
20 | ["0", 201, "0", 202, "0", 203, "0", 204, 205, 206, 207]
562+
(14 rows)
563+
564+
update test_jsonxa_arr set js = json_modify(js, set '$[0 to 300]' = '0');
565+
select id, json_query(js, '$[290 to 310]' with wrapper) from test_jsonxa_arr order by id;
566+
id | json_query
567+
----+-----------------------------------------------------------------------------------------------------------
568+
7 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0"]
569+
8 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0"]
570+
9 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
571+
10 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
572+
11 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
573+
12 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
574+
13 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
575+
14 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
576+
15 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
577+
16 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
578+
17 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
579+
18 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
580+
19 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
581+
20 | ["0", "0", "0", "0", "0", "0", "0", "0", "0", "0", "0", 298, 299, 300, 301, 302, 303, 304, 305, 306, 307]
582+
(14 rows)
583+
584+
update test_jsonxa_arr set js = json_modify(js, insert '$[1000003 to 1000005]' = '0');
585+
select id, json_query(js, '$[1000000 to 1000010]' with wrapper) from test_jsonxa_arr order by id;
586+
id | json_query
587+
----+--------------------------------------------------------------------------------------
588+
7 | [null, null, null, "0", "0", "0"]
589+
8 | [null, null, null, "0", "0", "0"]
590+
9 | [null, null, null, "0", "0", "0"]
591+
10 | [null, null, null, "0", "0", "0"]
592+
11 | [null, null, null, "0", "0", "0"]
593+
12 | [null, null, null, "0", "0", "0"]
594+
13 | [null, null, null, "0", "0", "0"]
595+
14 | [null, null, null, "0", "0", "0"]
596+
15 | [null, null, null, "0", "0", "0"]
597+
16 | [null, null, null, "0", "0", "0"]
598+
17 | [null, null, null, "0", "0", "0"]
599+
18 | [null, null, null, "0", "0", "0"]
600+
19 | [null, null, null, "0", "0", "0"]
601+
20 | [999997, 999998, 999999, "0", 1000000, "0", 1000001, "0", 1000002, 1000003, 1000004]
602+
(14 rows)
603+

‎contrib/jsonb_toaster/jsonb_toast_internals.c

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -998,7 +998,9 @@ jsonx_create_fetch_datum_iterator(struct varlena *attr, Oid toasterid,
998998
iter->toasterid = toasterid;
999999
iter->chunk_tids_inline_size = inline_size;
10001000

1001-
if (inline_size <= 0 ||
1001+
if (type == JSONX_PLAIN_JSONB ||
1002+
type == JSONX_POINTER ||
1003+
type == JSONX_POINTER_COMPRESSED_CHUNKS ||
10021004
type == JSONX_POINTER_DIFF ||
10031005
type == JSONX_POINTER_DIFF_COMP)
10041006
{
@@ -1009,8 +1011,11 @@ jsonx_create_fetch_datum_iterator(struct varlena *attr, Oid toasterid,
10091011
type == JSONX_POINTER_COMPRESSED_CHUNKS ||
10101012
type == JSONX_POINTER_DIFF_COMP;
10111013
}
1012-
else
1014+
else if (type == JSONX_POINTER_DIRECT_TIDS_COMP ||
1015+
type == JSONX_POINTER_DIRECT_TIDS)
10131016
{
1017+
Assert(inline_size > 0);
1018+
10141019
iter->nchunk_tids = header & ~JSONX_POINTER_TYPE_MASK;
10151020
iter->compressed_chunks = false;
10161021

‎contrib/jsonb_toaster/jsonb_toaster.h

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#define JSONX_POINTER_COMPRESSED_CHUNKS 0x40000000
2525
#define JSONX_POINTER_DIFF 0x50000000
2626
#define JSONX_POINTER_DIFF_COMP 0x60000000
27+
#define JSONX_CHUCKED_ARRAY 0x70000000
2728

2829
#define JSONX_CUSTOM_PTR_HEADER_SIZE (INTALIGN(VARATT_CUSTOM_SIZE(0)) + sizeof(uint32))
2930

@@ -119,13 +120,33 @@ typedef struct JsonxPointerDiff
119120
char data[FLEXIBLE_ARRAY_MEMBER];
120121
} JsonxPointerDiff;
121122

123+
#define JSONXA_INLINE_CHUNK 0x80000000
124+
125+
typedef ItemPointerData JsonxArrayChunkPtr;
126+
typedef uint32 JsonxArrayChunkOffset;
127+
128+
typedef struct JsonxArray
129+
{
130+
int32 n_elems;
131+
int32 n_chunks;
132+
Oid toastrelid;
133+
JsonxArrayChunkOffset chunk_offsets[FLEXIBLE_ARRAY_MEMBER];
134+
/* JsonxArrayChunkPtr chunk_ptrs[FLEXIBLE_ARRAY_MEMBER]; */
135+
} JsonxArray;
136+
137+
#define JSONX_ARRAY_HDR_SIZE (JSONX_CUSTOM_PTR_HEADER_SIZE + offsetof(JsonxArray, chunk_offsets))
138+
139+
extern JsonContainerOps jsonxaContainerOps;
140+
extern void jsonxaInit(JsonContainerData *jc, Datum value);
141+
122142
extern Datum jsonx_toast_save_datum(Relation rel, Datum value,
123143
struct varlena *oldexternal,
124144
int options);
125145
extern Datum
126146
jsonx_toast_save_datum_ext(Relation rel, Oid toasterid, Datum value,
127147
struct varlena *oldexternal, int options,
128148
struct varlena **p_chunk_tids,
149+
ItemPointerData *chunk_tids,
129150
bool compress_chunks);
130151
extern void jsonx_toast_delete_datum(Datum value, bool is_speculative);
131152

@@ -145,6 +166,17 @@ jsonx_toast_make_pointer_diff(Oid toasterid, struct varatt_external *ptr,
145166
int32 diff_offset, int32 diff_len,
146167
const void *diff_data);
147168

169+
extern Datum jsonx_toast_array_chunks(Relation rel, Oid toastrelid,
170+
Oid toasterid, int options,
171+
int n_elems, int n_chunks,
172+
JsonxArrayChunkOffset *chunk_offsets,
173+
JsonxArrayChunkPtr *chunk_ptrs,
174+
Datum *chunk_jbs);
175+
176+
extern struct varlena *
177+
jsonx_toast_wrap_array_into_pointer(Oid toasterid, JsonxArray *array,
178+
int data_size);
179+
148180
extern struct varlena *
149181
jsonx_toast_compress_tids(struct varlena *chunk_tids, int max_size);
150182

0 commit comments

Comments
 (0)