Skip to content

Commit 06e279a

Browse files
benjefferybenjeffery
committed
Add struct codec to the binary side
1 parent b75b870 commit 06e279a

File tree

6 files changed

+280
-156
lines changed

6 files changed

+280
-156
lines changed

‎c/tests/test_core.c‎

Lines changed: 34 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ set_u64_le(uint8_t *dest, uint64_t value)
9797
}
9898

9999
static void
100-
test_json_binary_metadata_get_blob(void)
100+
test_json_struct_metadata_get_blob(void)
101101
{
102102
int ret;
103103
char metadata[128];
@@ -132,7 +132,7 @@ test_json_binary_metadata_get_blob(void)
132132
memcpy(bytes + header_length, json_payload, json_length);
133133
memcpy(bytes + header_length + json_length, binary_payload, payload_length);
134134
metadata_length = (tsk_size_t) total_length;
135-
ret = tsk_json_binary_metadata_get_blob(
135+
ret = tsk_json_struct_metadata_get_blob(
136136
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
137137
CU_ASSERT_EQUAL(ret, 0);
138138
CU_ASSERT_PTR_EQUAL(json, (const char *) bytes + header_length);
@@ -149,7 +149,7 @@ test_json_binary_metadata_get_blob(void)
149149
CU_ASSERT_FATAL(total_length <= sizeof(metadata));
150150
set_u64_le(bytes + 13, (uint64_t) payload_length);
151151
metadata_length = (tsk_size_t) total_length;
152-
ret = tsk_json_binary_metadata_get_blob(
152+
ret = tsk_json_struct_metadata_get_blob(
153153
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
154154
CU_ASSERT_EQUAL(ret, 0);
155155
CU_ASSERT_PTR_EQUAL(json, (const char *) bytes + header_length);
@@ -165,7 +165,7 @@ test_json_binary_metadata_get_blob(void)
165165
set_u64_le(bytes + 13, (uint64_t) payload_length);
166166
memcpy(bytes + header_length + json_length, empty_payload, payload_length);
167167
metadata_length = (tsk_size_t) total_length;
168-
ret = tsk_json_binary_metadata_get_blob(
168+
ret = tsk_json_struct_metadata_get_blob(
169169
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
170170
CU_ASSERT_EQUAL(ret, 0);
171171
CU_ASSERT_PTR_EQUAL(json, (const char *) bytes + header_length);
@@ -179,43 +179,63 @@ test_json_binary_metadata_get_blob(void)
179179
json = NULL;
180180
json_buffer_length = 0;
181181
metadata_length = header_length - 1;
182-
ret = tsk_json_binary_metadata_get_blob(
182+
ret = tsk_json_struct_metadata_get_blob(
183183
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
184184
CU_ASSERT_EQUAL(ret, TSK_ERR_FILE_FORMAT);
185185

186186
metadata_length = (tsk_size_t) total_length;
187187
bytes[0] = 'X';
188-
ret = tsk_json_binary_metadata_get_blob(
188+
ret = tsk_json_struct_metadata_get_blob(
189189
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
190190
CU_ASSERT_EQUAL(ret, TSK_ERR_FILE_FORMAT);
191191
bytes[0] = 'J';
192192

193193
bytes[4] = 2;
194-
ret = tsk_json_binary_metadata_get_blob(
194+
ret = tsk_json_struct_metadata_get_blob(
195195
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
196196
CU_ASSERT_EQUAL(ret, TSK_ERR_FILE_VERSION_TOO_NEW);
197197
bytes[4] = 1;
198198

199199
metadata_length = (tsk_size_t)(total_length - 1);
200-
ret = tsk_json_binary_metadata_get_blob(
200+
ret = tsk_json_struct_metadata_get_blob(
201201
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
202202
CU_ASSERT_EQUAL(ret, TSK_ERR_FILE_FORMAT);
203203

204-
ret = tsk_json_binary_metadata_get_blob(
204+
ret = tsk_json_struct_metadata_get_blob(
205205
NULL, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
206206
CU_ASSERT_EQUAL(ret, TSK_ERR_BAD_PARAM_VALUE);
207-
ret = tsk_json_binary_metadata_get_blob(
207+
ret = tsk_json_struct_metadata_get_blob(
208208
metadata, metadata_length, NULL, &json_buffer_length, &blob, &blob_length);
209209
CU_ASSERT_EQUAL(ret, TSK_ERR_BAD_PARAM_VALUE);
210-
ret = tsk_json_binary_metadata_get_blob(
210+
ret = tsk_json_struct_metadata_get_blob(
211211
metadata, metadata_length, &json, NULL, &blob, &blob_length);
212212
CU_ASSERT_EQUAL(ret, TSK_ERR_BAD_PARAM_VALUE);
213-
ret = tsk_json_binary_metadata_get_blob(
213+
ret = tsk_json_struct_metadata_get_blob(
214214
metadata, metadata_length, &json, &json_buffer_length, NULL, &blob_length);
215215
CU_ASSERT_EQUAL(ret, TSK_ERR_BAD_PARAM_VALUE);
216-
ret = tsk_json_binary_metadata_get_blob(
216+
ret = tsk_json_struct_metadata_get_blob(
217217
metadata, metadata_length, &json, &json_buffer_length, &blob, NULL);
218218
CU_ASSERT_EQUAL(ret, TSK_ERR_BAD_PARAM_VALUE);
219+
220+
memset(metadata, 0, sizeof(metadata));
221+
bytes[0] = 'J';
222+
bytes[1] = 'B';
223+
bytes[2] = 'L';
224+
bytes[3] = 'B';
225+
bytes[4] = 1;
226+
metadata_length = (tsk_size_t) header_length;
227+
228+
set_u64_le(bytes + 5, UINT64_MAX - (uint64_t) header_length + 1);
229+
set_u64_le(bytes + 13, 0);
230+
ret = tsk_json_struct_metadata_get_blob(
231+
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
232+
CU_ASSERT_EQUAL(ret, TSK_ERR_FILE_FORMAT);
233+
234+
set_u64_le(bytes + 5, 8);
235+
set_u64_le(bytes + 13, UINT64_MAX - (uint64_t)(header_length + 8) + 1);
236+
ret = tsk_json_struct_metadata_get_blob(
237+
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
238+
CU_ASSERT_EQUAL(ret, TSK_ERR_FILE_FORMAT);
219239
}
220240

221241
static void
@@ -788,7 +808,7 @@ main(int argc, char **argv)
788808
{ "test_strerror", test_strerror },
789809
{ "test_strerror_kastore", test_strerror_kastore },
790810
{ "test_generate_uuid", test_generate_uuid },
791-
{ "test_json_binary_metadata_get_blob", test_json_binary_metadata_get_blob },
811+
{ "test_json_struct_metadata_get_blob", test_json_struct_metadata_get_blob },
792812
{ "test_double_round", test_double_round },
793813
{ "test_blkalloc", test_blkalloc },
794814
{ "test_unknown_time", test_unknown_time },

‎c/tskit/core.c‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -142,14 +142,16 @@ tsk_generate_uuid(char *dest, int TSK_UNUSED(flags))
142142
}
143143

144144
int
145-
tsk_json_binary_metadata_get_blob(const char *metadata, tsk_size_t metadata_length,
145+
tsk_json_struct_metadata_get_blob(const char *metadata, tsk_size_t metadata_length,
146146
const char **json, tsk_size_t *json_length, const uint8_t **blob,
147147
tsk_size_t *blob_length)
148148
{
149149
int ret;
150150
uint8_t version;
151151
uint64_t json_length_u64;
152152
uint64_t binary_length_u64;
153+
uint64_t header_and_json_length;
154+
uint64_t total_length;
153155
const uint8_t *bytes;
154156
const uint8_t *blob_start;
155157
const char *json_start;
@@ -175,8 +177,17 @@ tsk_json_binary_metadata_get_blob(const char *metadata, tsk_size_t metadata_leng
175177
}
176178
json_length_u64 = tsk_load_u64_le(bytes + 5);
177179
binary_length_u64 = tsk_load_u64_le(bytes + 13);
178-
if ((uint64_t) metadata_length
179-
< (uint64_t) TSK_JSON_BINARY_HEADER_SIZE + json_length_u64 + binary_length_u64) {
180+
if (json_length_u64 > UINT64_MAX - (uint64_t) TSK_JSON_BINARY_HEADER_SIZE) {
181+
ret = tsk_trace_error(TSK_ERR_FILE_FORMAT);
182+
goto out;
183+
}
184+
header_and_json_length = (uint64_t) TSK_JSON_BINARY_HEADER_SIZE + json_length_u64;
185+
if (binary_length_u64 > UINT64_MAX - header_and_json_length) {
186+
ret = tsk_trace_error(TSK_ERR_FILE_FORMAT);
187+
goto out;
188+
}
189+
total_length = header_and_json_length + binary_length_u64;
190+
if ((uint64_t) metadata_length < total_length) {
180191
ret = tsk_trace_error(TSK_ERR_FILE_FORMAT);
181192
goto out;
182193
}

‎c/tskit/core.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1097,13 +1097,13 @@ bool tsk_isfinite(double val);
10971097
int tsk_generate_uuid(char *dest, int flags);
10981098

10991099
/**
1100-
@brief Extract the binary payload from ``json+binary`` encoded metadata.
1100+
@brief Extract the binary payload from ``json+struct`` encoded metadata.
11011101
11021102
@rst
1103-
Metadata produced by :py:class:`tskit.metadata.JSONBinaryCodec` consists of a fixed-size
1103+
Metadata produced by :py:class:`tskit.metadata.JSONStructCodec` consists of a fixed-size
11041104
header followed by canonical JSON bytes and an optional binary payload. This helper
1105-
validates the ``json+binary`` framing, returning pointers to the embedded JSON and binary
1106-
sections without copying.
1105+
validates the framing, returning pointers to the embedded JSON and binary sections
1106+
without copying.
11071107
11081108
The output pointers reference memory owned by the caller and remain valid only while
11091109
the original metadata buffer is alive.
@@ -1117,7 +1117,7 @@ the original metadata buffer is alive.
11171117
@param[out] blob_length On success, set to the payload length in bytes.
11181118
@return 0 on success, or a :ref:`TSK_ERR <c_api_errors>` code on failure.
11191119
*/
1120-
int tsk_json_binary_metadata_get_blob(const char *metadata, tsk_size_t metadata_length,
1120+
int tsk_json_struct_metadata_get_blob(const char *metadata, tsk_size_t metadata_length,
11211121
const char **json, tsk_size_t *json_length, const uint8_t **blob,
11221122
tsk_size_t *blob_length);
11231123

‎python/CHANGELOG.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,8 @@
8787
also around 10% faster.
8888
(:user:`benjeffery`, :pr:`3313`, :pr:`3317`, :issue:`1896`)
8989

90-
- Add ``json+binary`` metadata codec that allows storing binary data
91-
alongside JSON metadata. (:user:`benjeffery`, :pr:`3306`)
90+
- Add ``json+struct`` metadata codec that allows storing binary data using a struct
91+
schema alongside JSON metadata. (:user:`benjeffery`, :pr:`3306`)
9292

9393
**Bugfixes**
9494

0 commit comments

Comments
 (0)