Skip to content

Commit ba0e1c2

Browse files
author
Nikita Glukhov
committed
Add compression options support to json
1 parent 90d68c7 commit ba0e1c2

File tree

5 files changed

+112
-49
lines changed

5 files changed

+112
-49
lines changed

‎src/backend/utils/adt/json.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@
7272

7373
#include "postgres.h"
7474

75+
#include "access/compression.h"
7576
#include "access/htup_details.h"
7677
#include "access/transam.h"
7778
#include "catalog/pg_type.h"
@@ -2627,7 +2628,7 @@ jsontInitContainer(JsonContainerData *jc, char *json, int len,
26272628
}
26282629

26292630
static void
2630-
jsontInit(JsonContainerData *jc, Datum value)
2631+
jsontInit(JsonContainerData *jc, Datum value, CompressionOptions options)
26312632
{
26322633
text *json = DatumGetTextP(value);
26332634
JsonLexContext *lex = makeJsonLexContext(json, false);
@@ -3027,6 +3028,7 @@ JsonContainerOps
30273028
jsontContainerOps =
30283029
{
30293030
JsonContainerJsont,
3031+
NULL,
30303032
jsontInit,
30313033
JsontIteratorInit,
30323034
jsonFindLastKeyInObject,

‎src/backend/utils/adt/json_generic.c

Lines changed: 57 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,10 @@
1313
#include "utils/json_generic.h"
1414
#include "utils/memutils.h"
1515

16-
static JsonContainerOps jsonvContainerOps;
16+
static Json *JsonExpand(Datum value, JsonContainerOps *ops,
17+
CompressionOptions opts);
1718

18-
static Json *JsonExpand(Datum value, JsonContainerOps *ops);
19+
static JsonContainerOps jsonvContainerOps;
1920

2021
#if 0
2122
static JsonValue *
@@ -544,6 +545,7 @@ jsonvContainerOps =
544545
{
545546
JsonContainerJsonv,
546547
NULL,
548+
NULL,
547549
jsonvIteratorInit,
548550
jsonvFindKeyInObject,
549551
jsonvFindValueInArray,
@@ -577,20 +579,24 @@ typedef struct varatt_extended_json
577579
{
578580
varatt_extended_hdr vaext;
579581
JsonContainerType type;
580-
char params[FLEXIBLE_ARRAY_MEMBER];
582+
char data[FLEXIBLE_ARRAY_MEMBER];
581583
} varatt_extended_json;
582584

583585
static Size
584586
jsonGetExtendedSize(JsonContainer *jc)
585587
{
586-
return VARHDRSZ_EXTERNAL + offsetof(varatt_extended_json, params) + jc->len;
588+
return VARHDRSZ_EXTERNAL + offsetof(varatt_extended_json, data) +
589+
VARHDRSZ + jc->len +
590+
(jc->ops->compressionOps ?
591+
jc->ops->compressionOps->encodeOptions(jc, NULL) : 0);
587592
}
588593

589594
static void
590595
jsonWriteExtended(JsonContainer *jc, void *ptr, Size allocated_size)
591596
{
592597
varatt_extended_json extjs,
593598
*pextjs;
599+
Size optionsSize;
594600

595601
Assert(allocated_size >= jsonGetExtendedSize(jc));
596602

@@ -600,8 +606,13 @@ jsonWriteExtended(JsonContainer *jc, void *ptr, Size allocated_size)
600606

601607
SET_VARTAG_EXTERNAL(ptr, VARTAG_EXTENDED);
602608
pextjs = (varatt_extended_json *) VARDATA_EXTERNAL(ptr);
603-
memcpy(pextjs, &extjs, offsetof(varatt_extended_json, params));
604-
memcpy(&pextjs->params, jc->data, jc->len);
609+
memcpy(pextjs, &extjs, offsetof(varatt_extended_json, data));
610+
611+
optionsSize = jc->ops->compressionOps ?
612+
jc->ops->compressionOps->encodeOptions(jc, &pextjs->data) : 0;
613+
614+
SET_VARSIZE(&pextjs->data[optionsSize], VARHDRSZ + jc->len);
615+
memcpy(VARDATA(&pextjs->data[optionsSize]), jc->data, jc->len);
605616
}
606617

607618
static Json *
@@ -612,37 +623,51 @@ JsonInitExtended(const struct varlena *toasted,
612623
CompressionMethodRoutine *cmr;
613624
varatt_extended_json *pextjs,
614625
extjs;
615-
void *val;
616-
int len;
617626
Datum value;
627+
CompressionOptions options;
628+
Size totalSize;
629+
Size optionsSize;
630+
Size (*decodeOptions)(const void *buf,
631+
CompressionOptions *options);
618632

619633
Assert(VARATT_IS_EXTERNAL_EXTENDED(detoasted));
620634

621635
pextjs = (varatt_extended_json *) VARDATA_EXTERNAL(detoasted);
622-
memcpy(&extjs, pextjs, offsetof(varatt_extended_json, params));
636+
memcpy(&extjs, pextjs, offsetof(varatt_extended_json, data));
637+
638+
totalSize = extjs.vaext.size - offsetof(varatt_extended_json, data);
623639

624640
ops = JsonContainerGetOpsByType(extjs.type);
625641

626642
if (ops)
643+
{
627644
cmr = NULL;
645+
decodeOptions = ops->compressionOps ?
646+
ops->compressionOps->decodeOptions : NULL;
647+
}
628648
else
629649
{
630650
cmr = GetCompressionMethodRoutine(extjs.type, InvalidOid);
631651

632652
if (!cmr)
633653
elog(ERROR, "unrecognized json container type %d", extjs.type);
654+
655+
decodeOptions = cmr->options ? cmr->options->decode : NULL;
634656
}
635657

636-
len = extjs.vaext.size - offsetof(varatt_extended_json, params);
658+
optionsSize = decodeOptions ? decodeOptions(&pextjs->data, &options) : 0;
637659

638-
val = palloc(VARHDRSZ + len); /* FIXME save value with varlena header */
639-
SET_VARSIZE(val, VARHDRSZ + len);
640-
memcpy(VARDATA(val), &pextjs->params, len);
660+
/* FIXME alignment
661+
* value = PointerGetDatum(&pextjs->data[optionsSize]);
662+
*/
663+
value = PointerGetDatum(memcpy(palloc(totalSize - optionsSize),
664+
&pextjs->data[optionsSize],
665+
totalSize - optionsSize));
641666

642667
if (ops)
643-
return JsonExpand(PointerGetDatum(val), ops);
668+
return JsonExpand(value, ops, options);
644669

645-
value = cmr->decompress(PointerGetDatum(val), NULL);
670+
value = cmr->decompress(value, options);
646671

647672
Assert(VARATT_IS_EXTERNAL_EXPANDED(DatumGetPointer(value)));
648673

@@ -665,7 +690,7 @@ JsonInit(Json *json)
665690
Assert(!VARATT_IS_EXTERNAL_EXTENDED(data));
666691
json->obj.compressed = PointerGetDatum(data);
667692

668-
(*json->root.ops->init)(&json->root, json->obj.compressed);
693+
json->root.ops->init(&json->root, json->obj.compressed, json->obj.options);
669694
}
670695

671696
static Size
@@ -822,7 +847,7 @@ jsonExpandedObjectMethods =
822847
};
823848

824849
static Json *
825-
JsonExpand(Datum value, JsonContainerOps *ops)
850+
JsonExpand(Datum value, JsonContainerOps *ops, CompressionOptions options)
826851
{
827852
MemoryContext objcxt;
828853
Json *json;
@@ -843,6 +868,7 @@ JsonExpand(Datum value, JsonContainerOps *ops)
843868
EOH_init_header(&json->obj.eoh, &jsonExpandedObjectMethods, objcxt);
844869

845870
json->obj.compressed = value;
871+
json->obj.options = options;
846872
json->root.data = NULL;
847873
json->root.len = 0;
848874
json->root.ops = ops;
@@ -853,8 +879,8 @@ JsonExpand(Datum value, JsonContainerOps *ops)
853879
return json;
854880
}
855881

856-
Json *
857-
DatumGetJson(Datum value, JsonContainerOps *ops)
882+
static Json *
883+
JsonExpandDatum(Datum value, JsonContainerOps *ops, CompressionOptions options)
858884
{
859885
struct varlena *toasted = (struct varlena *) DatumGetPointer(value);
860886
Json *json;
@@ -872,11 +898,17 @@ DatumGetJson(Datum value, JsonContainerOps *ops)
872898
elog(ERROR, "unexpected extended json");
873899
#endif
874900

875-
json = JsonExpand(PointerGetDatum(detoasted), ops);
901+
json = JsonExpand(PointerGetDatum(detoasted), ops, options);
876902
}
877903

878-
JsonInit(json);
904+
return json;
905+
}
879906

907+
Json *
908+
DatumGetJson(Datum value, JsonContainerOps *ops, CompressionOptions options)
909+
{
910+
Json *json = JsonExpandDatum(value, ops, options);
911+
JsonInit(json);
880912
return json;
881913
}
882914

@@ -885,13 +917,14 @@ JsonValueToJson(JsonValue *val)
885917
{
886918
if (val->type == jbvBinary)
887919
{
888-
Json *json = JsonExpand(PointerGetDatum(NULL), NULL);
889-
json->root = *val->val.binary.data;
920+
JsonContainer *jc = val->val.binary.data;
921+
Json *json = JsonExpand(PointerGetDatum(NULL), jc->ops, NULL);
922+
json->root = *jc;
890923
return json;
891924
}
892925
else
893926
{
894-
Json *json = JsonExpand(PointerGetDatum(NULL), &jsonvContainerOps);
927+
Json *json = JsonExpand(PointerGetDatum(NULL), &jsonvContainerOps, NULL);
895928
jsonvInitContainer(&json->root, val);
896929
return json;
897930
}

‎src/backend/utils/adt/jsonb_util.c

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
*/
1414
#include "postgres.h"
1515

16+
#include "access/compression.h"
1617
#include "access/hash.h"
1718
#include "catalog/pg_collation.h"
1819
#include "miscadmin.h"
@@ -78,7 +79,8 @@ static void fillJsonbValue(const JsonbContainer *container, int index,
7879
char *base_addr, uint32 offset,
7980
JsonbValue *result);
8081
static int compareJsonbScalarValue(const JsonbValue *a, const JsonbValue *b);
81-
static void *convertToJsonb(const JsonbValue *val, JsonValueEncoder encoder);
82+
static void *convertToJsonb(const JsonbValue *val, JsonValueEncoder encoder,
83+
CompressionOptions options);
8284
static void convertJsonbValue(StringInfo buffer, JEntry *header, const JsonbValue *val, int level);
8385
static void convertJsonbArray(StringInfo buffer, JEntry *header, const JsonbValue *val, int level);
8486
static void convertJsonbObject(StringInfo buffer, JEntry *header, const JsonbValue *val, int level);
@@ -112,11 +114,14 @@ JsonValueUnpackBinary(const JsonValue *jbv)
112114

113115
void *
114116
JsonContainerFlatten(JsonContainer *jc, JsonValueEncoder encoder,
115-
JsonContainerOps *ops, const JsonValue *binary)
117+
JsonContainerOps *ops, CompressionOptions options,
118+
const JsonValue *binary)
116119
{
117120
JsonbValue jbv;
118121

119-
if (jc->ops == ops)
122+
if (jc->ops == ops &&
123+
(!jc->ops->compressionOps ||
124+
jc->ops->compressionOps->optionsAreEqual(jc, options)))
120125
{
121126
int size = jc->len;
122127
void *out = palloc(VARHDRSZ + size);
@@ -135,7 +140,7 @@ JsonContainerFlatten(JsonContainer *jc, JsonValueEncoder encoder,
135140
binary = &jbv;
136141
}
137142

138-
return convertToJsonb(binary, encoder);
143+
return convertToJsonb(binary, encoder, options);
139144
}
140145

141146
/*
@@ -153,10 +158,11 @@ JsonContainerFlatten(JsonContainer *jc, JsonValueEncoder encoder,
153158
*/
154159
void *
155160
JsonValueFlatten(const JsonValue *val, JsonValueEncoder encoder,
156-
JsonContainerOps *ops)
161+
JsonContainerOps *ops, CompressionOptions options)
157162
{
158163
if (val->type == jbvBinary)
159-
return JsonContainerFlatten(val->val.binary.data, encoder, ops, val);
164+
return JsonContainerFlatten(val->val.binary.data, encoder, ops, options,
165+
val);
160166

161167
if (IsAJsonbScalar(val))
162168
{
@@ -168,7 +174,7 @@ JsonValueFlatten(const JsonValue *val, JsonValueEncoder encoder,
168174
Assert(val->type == jbvObject || val->type == jbvArray);
169175
}
170176

171-
return convertToJsonb(val, encoder);
177+
return convertToJsonb(val, encoder, options);
172178
}
173179

174180
/*
@@ -1580,7 +1586,8 @@ padBufferToInt(StringInfo buffer)
15801586
}
15811587

15821588
void
1583-
JsonbEncode(StringInfoData *buffer, const JsonbValue *val)
1589+
JsonbEncode(StringInfoData *buffer, const JsonbValue *val,
1590+
CompressionOptions options)
15841591
{
15851592
JEntry jentry;
15861593

@@ -1591,7 +1598,8 @@ JsonbEncode(StringInfoData *buffer, const JsonbValue *val)
15911598
* Given a JsonbValue, convert to Jsonb. The result is palloc'd.
15921599
*/
15931600
static void *
1594-
convertToJsonb(const JsonbValue *val, JsonValueEncoder encoder)
1601+
convertToJsonb(const JsonbValue *val, JsonValueEncoder encoder,
1602+
CompressionOptions options)
15951603
{
15961604
StringInfoData buffer;
15971605
void *res;
@@ -1602,7 +1610,7 @@ convertToJsonb(const JsonbValue *val, JsonValueEncoder encoder)
16021610
/* Make room for the varlena header */
16031611
reserveFromBuffer(&buffer, VARHDRSZ);
16041612

1605-
(*encoder)(&buffer, val);
1613+
(*encoder)(&buffer, val, options);
16061614

16071615
/*
16081616
* Note: the JEntry of the root is discarded. Therefore the root
@@ -2039,7 +2047,7 @@ jsonbInitContainer(JsonContainerData *jc, JsonbContainer *jbc, int len)
20392047
}
20402048

20412049
static void
2042-
jsonbInit(JsonContainerData *jc, Datum value)
2050+
jsonbInit(JsonContainerData *jc, Datum value, CompressionOptions options)
20432051
{
20442052
Jsonb *jb = (Jsonb *) DatumGetPointer(value);
20452053
jsonbInitContainer(jc, &jb->root, VARSIZE_ANY_EXHDR(jb));
@@ -2049,6 +2057,7 @@ JsonContainerOps
20492057
jsonbContainerOps =
20502058
{
20512059
JsonContainerJsonb,
2060+
NULL,
20522061
jsonbInit,
20532062
JsonbIteratorInit,
20542063
jsonbFindKeyInObject,

‎src/include/access/compression.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ typedef struct CompressionMethodOptionsRoutines
2727
void (*free)(CompressionOptions options);
2828
CompressionOptions (*copy)(CompressionOptions options);
2929
bool (*equal)(CompressionOptions o1, CompressionOptions o2);
30+
Size (*decode)(const void *buf, CompressionOptions *options);
3031
} CompressionMethodOptionsRoutines;
3132

3233
typedef Datum (*CompressionRoutine) (Datum value, CompressionOptions options);

0 commit comments

Comments
 (0)