From 1dd398ff1a63b00b8934c9d36ba58e39dc85106e Mon Sep 17 00:00:00 2001 From: Greg Richardson Date: Wed, 30 Apr 2025 16:25:56 -0600 Subject: [PATCH 1/2] feat: protobuf + json poc --- packages/pg-parser/Makefile | 74 +++++++++++++-- packages/pg-parser/bindings/Filelists.mk | 3 +- packages/pg-parser/bindings/parse.c | 10 +- packages/pg-parser/bindings/protobuf-json.c | 63 ++++++++++++ packages/pg-parser/src/pg-parser.test.ts | 51 ++++++---- packages/pg-parser/src/pg-parser.ts | 100 +++++++++++++------- packages/pg-parser/tools/emsdk/Dockerfile | 5 + 7 files changed, 235 insertions(+), 71 deletions(-) create mode 100644 packages/pg-parser/bindings/protobuf-json.c diff --git a/packages/pg-parser/Makefile b/packages/pg-parser/Makefile index 38b83b0..6a4d7e8 100644 --- a/packages/pg-parser/Makefile +++ b/packages/pg-parser/Makefile @@ -1,3 +1,9 @@ +AUTOCONF = autoconf +AUTOMAKE = automake +ACLOCAL = aclocal +AUTORECONF = autoreconf +AUTORECONF_FLAGS = -fiv + PROTOBUF_TYPE_GENERATOR = tsx scripts/generate-types.ts SRC_DIR = bindings @@ -5,6 +11,7 @@ OUTPUT_DIR = wasm/$(LIBPG_QUERY_VERSION) OUTPUT_JS = $(OUTPUT_DIR)/pg-parser.js OUTPUT_WASM = $(OUTPUT_DIR)/pg-parser.wasm OUTPUT_D_TS = pg-parser.d.ts +OUTPUT_FILES = $(OUTPUT_JS) $(OUTPUT_WASM) $(OUTPUT_D_TS) WASM_MODULE_NAME := PgParserModule include $(SRC_DIR)/Filelists.mk @@ -12,11 +19,14 @@ OBJ_FILES = $(SRC_FILES:.c=.o) INCLUDE = $(SRC_DIR)/include CFLAGS = -Oz -Wall -std=c11 -LDFLAGS = -Wl,--gc-sections,--strip-all \ +LDFLAGS = -Wl,--gc-sections,--strip-all + +EMSCRIPTEN_FLAGS = \ --no-entry \ - -sFILESYSTEM=0 \ + -sFILESYSTEM=1 \ + -sALLOW_MEMORY_GROWTH=1 \ -sEXPORT_NAME="$(WASM_MODULE_NAME)" \ - -sEXPORTED_RUNTIME_METHODS=ccall,cwrap,getValue,UTF8ToString \ + -sEXPORTED_RUNTIME_METHODS=ccall,cwrap,getValue,UTF8ToString,HEAPU8 \ -sMODULARIZE=1 \ -sEXPORT_ES6=1 @@ -29,25 +39,71 @@ LIBPG_QUERY_SRC_DIR = $(LIBPG_QUERY_DIR)/src LIBPG_QUERY_LIB = $(LIBPG_QUERY_DIR)/libpg_query.a LIBPG_QUERY_STAMP = $(LIBPG_QUERY_DIR)/.stamp LIBPG_QUERY_VERSION := $(firstword $(subst -, ,$(LIBPG_QUERY_TAG))) +LIBPG_QUERY_PROTOBUF_DIR = $(LIBPG_QUERY_DIR)/protobuf +LIBPG_QUERY_PROTOBUF_C_DIR = $(LIBPG_QUERY_DIR)/vendor/protobuf-c +LIBPG_QUERY_PROTOBUF_C_SRC_FILES = $(wildcard $(LIBPG_QUERY_PROTOBUF_C_DIR)/*.c) +LIBPG_QUERY_PROTOBUF_C_OBJ_FILES = $(LIBPG_QUERY_PROTOBUF_C_SRC_FILES:.c=.o) + +PROTOBUF_2_JSON_REPO = https://github.com/Sannis/protobuf2json-c.git +PROTOBUF_2_JSON_TAG = v0.4.0 +PROTOBUF_2_JSON_DIR = $(VENDOR_DIR)/protobuf2json-c/$(PROTOBUF_2_JSON_TAG) +PROTOBUF_2_JSON_SRC_DIR = $(PROTOBUF_2_JSON_DIR)/src +PROTOBUF_2_JSON_INCLUDE = $(PROTOBUF_2_JSON_DIR)/include +PROTOBUF_2_JSON_LIB = $(PROTOBUF_2_JSON_DIR)/protobuf2json.a +PROTOBUF_2_JSON_STAMP = $(PROTOBUF_2_JSON_DIR)/.stamp +PROTOBUF_2_JSON_FILES = $(wildcard $(PROTOBUF_2_JSON_SRC_DIR)/*.c) +PROTOBUF_2_JSON_OBJ_FILES = $(PROTOBUF_2_JSON_FILES:.c=.o) + +JANSSON_REPO = https://github.com/akheron/jansson.git +JANSSON_TAG = v2.14.1 +JANSSON_DIR = $(VENDOR_DIR)/jansson/$(JANSSON_TAG) +JANSSON_SRC_DIR = $(JANSSON_DIR)/src +JANSSON_LIB = $(JANSSON_SRC_DIR)/.libs/libjansson.a +JANSSON_STAMP = $(JANSSON_DIR)/.stamp .DEFAULT_GOAL := build -$(OUTPUT_JS): $(OBJ_FILES) $(LIBPG_QUERY_LIB) +$(OUTPUT_FILES): $(OBJ_FILES) $(LIBPG_QUERY_LIB) $(PROTOBUF_2_JSON_LIB) $(JANSSON_LIB) @mkdir -p $(OUTPUT_DIR) - $(CC) $(LDFLAGS) -L$(LIBPG_QUERY_DIR) -lpg_query -o $(OUTPUT_JS) $(OBJ_FILES) --closure 0 --emit-tsd $(OUTPUT_D_TS) + $(CC) $(LDFLAGS) $(EMSCRIPTEN_FLAGS) -o $(OUTPUT_JS) $(OBJ_FILES) $(LIBPG_QUERY_LIB) $(JANSSON_LIB) $(PROTOBUF_2_JSON_LIB) --closure 0 --emit-tsd $(OUTPUT_D_TS) $(PROTOBUF_TYPE_GENERATOR) -i $(LIBPG_QUERY_DIR)/protobuf/pg_query.proto -o $(OUTPUT_DIR) -%.o: %.c | $(LIBPG_QUERY_LIB) - $(CC) -I$(LIBPG_QUERY_DIR) -I$(INCLUDE) $(CFLAGS) -c $< -o $@ +$(OBJ_FILES): %.o: %.c | $(LIBPG_QUERY_LIB) $(PROTOBUF_2_JSON_LIB) $(JANSSON_LIB) + $(CC) -I$(LIBPG_QUERY_DIR) -I$(LIBPG_QUERY_DIR)/vendor -I$(PROTOBUF_2_JSON_INCLUDE) -I$(JANSSON_SRC_DIR) -I$(INCLUDE) $(CFLAGS) -c $< -o $@ + +$(PROTOBUF_2_JSON_OBJ_FILES): %.o: %.c $(PROTOBUF_2_JSON_INCLUDE) $(JANSSON_LIB) + echo "Compiling $<" + $(CC) -I$(PROTOBUF_2_JSON_INCLUDE) -I$(JANSSON_SRC_DIR) $(CFLAGS) -c $< -o $@ $(LIBPG_QUERY_LIB): $(LIBPG_QUERY_STAMP) $(MAKE) -C $(LIBPG_QUERY_DIR) build +$(PROTOBUF_2_JSON_LIB): $(PROTOBUF_2_JSON_STAMP) $(PROTOBUF_2_JSON_OBJ_FILES) $(JANSSON_LIB) $(LIBPG_QUERY_PROTOBUF_C_OBJ_FILES) + $(AR) rcs $(PROTOBUF_2_JSON_LIB) $(PROTOBUF_2_JSON_OBJ_FILES) + +$(JANSSON_LIB): $(JANSSON_STAMP) + cd $(JANSSON_DIR) && \ + $(AUTORECONF) -i && \ + emconfigure ./configure --host=wasm32 && \ + $(MAKE) + $(LIBPG_QUERY_STAMP): - git clone --depth 1 --branch $(LIBPG_QUERY_TAG) $(LIBPG_QUERY_REPO) $(LIBPG_QUERY_DIR) + git clone -c advice.detachedHead=false --depth 1 --branch $(LIBPG_QUERY_TAG) $(LIBPG_QUERY_REPO) $(LIBPG_QUERY_DIR) + touch $@ + +$(PROTOBUF_2_JSON_STAMP): $(LIBPG_QUERY_STAMP) $(JANSSON_LIB) + git clone -c advice.detachedHead=false --depth 1 --branch $(PROTOBUF_2_JSON_TAG) $(PROTOBUF_2_JSON_REPO) $(PROTOBUF_2_JSON_DIR) + touch $@ + @mkdir -p $(PROTOBUF_2_JSON_INCLUDE)/google + ln -s ../../../../../$(LIBPG_QUERY_PROTOBUF_C_DIR) $(PROTOBUF_2_JSON_INCLUDE)/google/protobuf-c + ln -s ../../../../$(JANSSON_SRC_DIR)/jansson.h $(PROTOBUF_2_JSON_INCLUDE)/jansson.h + ln -s ../../../../$(JANSSON_SRC_DIR)/jansson_config.h $(PROTOBUF_2_JSON_INCLUDE)/jansson_config.h + +$(JANSSON_STAMP): + git clone -c advice.detachedHead=false --depth 1 --branch $(JANSSON_TAG) $(JANSSON_REPO) $(JANSSON_DIR) touch $@ -build: $(OUTPUT_JS) +build: $(OUTPUT_FILES) clean: rm -rf $(OUTPUT_DIR) diff --git a/packages/pg-parser/bindings/Filelists.mk b/packages/pg-parser/bindings/Filelists.mk index ebfde6e..205a797 100644 --- a/packages/pg-parser/bindings/Filelists.mk +++ b/packages/pg-parser/bindings/Filelists.mk @@ -1,2 +1,3 @@ SRC_FILES= \ - $(SRC_DIR)/parse.c \ No newline at end of file + $(SRC_DIR)/parse.c \ + $(SRC_DIR)/protobuf-json.c \ No newline at end of file diff --git a/packages/pg-parser/bindings/parse.c b/packages/pg-parser/bindings/parse.c index c2d130b..26238b2 100644 --- a/packages/pg-parser/bindings/parse.c +++ b/packages/pg-parser/bindings/parse.c @@ -3,16 +3,16 @@ #include "macros.h" EXPORT("parse_sql") -PgQueryParseResult *parse_sql(char *sql) +PgQueryProtobufParseResult *parse_sql(char *sql) { - PgQueryParseResult *result = (PgQueryParseResult *)malloc(sizeof(PgQueryParseResult)); - *result = pg_query_parse(sql); + PgQueryProtobufParseResult *result = (PgQueryProtobufParseResult *)malloc(sizeof(PgQueryProtobufParseResult)); + *result = pg_query_parse_protobuf(sql); return result; } EXPORT("free_parse_result") -void free_parse_result(PgQueryParseResult *result) +void free_parse_result(PgQueryProtobufParseResult *result) { - pg_query_free_parse_result(*result); + pg_query_free_protobuf_parse_result(*result); free(result); } \ No newline at end of file diff --git a/packages/pg-parser/bindings/protobuf-json.c b/packages/pg-parser/bindings/protobuf-json.c new file mode 100644 index 0000000..6efdd28 --- /dev/null +++ b/packages/pg-parser/bindings/protobuf-json.c @@ -0,0 +1,63 @@ +#define _POSIX_C_SOURCE 200809L + +#include +#include +#include + +#include "pg_query.h" +#include "protobuf2json.h" +#include "protobuf/pg_query.pb-c.h" +#include "macros.h" + +typedef struct +{ + char *json_string; + char *error; +} ProtobufToJsonResult; + +EXPORT("protobuf_to_json") +ProtobufToJsonResult *protobuf_to_json(PgQueryProtobuf *protobuf) +{ + ProtobufToJsonResult *result = (ProtobufToJsonResult *)malloc(sizeof(ProtobufToJsonResult)); + + // Unpack the protobuf binary data back into a message + PgQuery__ParseResult *parse_result = pg_query__parse_result__unpack(NULL, protobuf->len, (uint8_t *)protobuf->data); + if (!parse_result) + { + result->error = strdup("Failed to unpack protobuf message"); + return result; + } + + char *error = (char *)malloc(256); + + // Convert the protobuf message to JSON + int ret = protobuf2json_string( + &parse_result->base, + 0, + &result->json_string, + error, + sizeof(*error)); + + if (ret != 0) + { + result->error = error; + return result; + } + + free(error); + return result; +} + +EXPORT("free_protobuf_to_json_result") +void free_protobuf_to_json_result(ProtobufToJsonResult *result) +{ + if (result->json_string) + { + free(result->json_string); + } + if (result->error) + { + free(result->error); + } + free(result); +} diff --git a/packages/pg-parser/src/pg-parser.test.ts b/packages/pg-parser/src/pg-parser.test.ts index ae5d466..4e61ca1 100644 --- a/packages/pg-parser/src/pg-parser.test.ts +++ b/packages/pg-parser/src/pg-parser.test.ts @@ -2,28 +2,42 @@ import { describe, expect, it } from 'vitest'; import { PgParser } from './pg-parser.js'; import { unwrapResult } from './util.js'; -describe('pg-parser', () => { - it('parses sql in v15', async () => { - const pgParser = new PgParser({ version: 15 }); - const result = await unwrapResult(pgParser.parseSql('SELECT 1+1 as sum')); - expect(result.version).toBe(150001); - }); - - it('parses sql in v16', async () => { - const pgParser = new PgParser({ version: 16 }); - const result = await unwrapResult(pgParser.parseSql('SELECT 1+1 as sum')); - expect(result.version).toBe(160001); - }); +describe('versions', () => { + // it('parses sql in v15', async () => { + // const pgParser = new PgParser({ version: 15 }); + // const result = await unwrapResult(pgParser.parse('SELECT 1+1 as sum')); + // expect(result.version).toBe(150001); + // }); + + // it('parses sql in v16', async () => { + // const pgParser = new PgParser({ version: 16 }); + // const result = await unwrapResult(pgParser.parse('SELECT 1+1 as sum')); + // expect(result.version).toBe(160001); + // }); it('parses sql in v17', async () => { const pgParser = new PgParser({ version: 17 }); - const result = await unwrapResult(pgParser.parseSql('SELECT 1+1 as sum')); + const result = await unwrapResult(pgParser.parse('SELECT 1+1 as sum')); + expect(result.version).toBe(170004); + }); + + it('parses sql in v17 by default', async () => { + const pgParser = new PgParser(); + const result = await unwrapResult(pgParser.parse('SELECT 1+1 as sum')); expect(result.version).toBe(170004); }); + it('throws error for unsupported version', async () => { + const create = () => new PgParser({ version: 13 as any }); + expect(create).toThrow('unsupported version'); + }); +}); + +describe('parser', () => { it('parses sql into ast', async () => { const pgParser = new PgParser(); - const result = await unwrapResult(pgParser.parseSql('SELECT 1+1 as sum')); + const result = await unwrapResult(pgParser.parse('SELECT 1+1 as sum')); + expect(result).toMatchObject({ stmts: [ { @@ -71,7 +85,7 @@ describe('pg-parser', () => { it('parse result matches types', async () => { const pgParser = new PgParser(); - const result = await unwrapResult(pgParser.parseSql('SELECT 1+1 as sum')); + const result = await unwrapResult(pgParser.parse('SELECT 1+1 as sum')); expect(result.version).toBe(170004); @@ -179,12 +193,7 @@ describe('pg-parser', () => { it('throws error for invalid sql', async () => { const pgParser = new PgParser(); - const resultPromise = unwrapResult(pgParser.parseSql('my invalid sql')); + const resultPromise = unwrapResult(pgParser.parse('my invalid sql')); await expect(resultPromise).rejects.toThrow('syntax error at or near "my"'); }); - - it('throws error for unsupported version', async () => { - const create = () => new PgParser({ version: 13 as any }); - expect(create).toThrow('unsupported version'); - }); }); diff --git a/packages/pg-parser/src/pg-parser.ts b/packages/pg-parser/src/pg-parser.ts index 0429137..d969c29 100644 --- a/packages/pg-parser/src/pg-parser.ts +++ b/packages/pg-parser/src/pg-parser.ts @@ -1,11 +1,6 @@ import { SUPPORTED_VERSIONS } from './constants.js'; import { PgParseError } from './errors.js'; -import type { - MainModule, - PgParseResult, - PgParserModule, - SupportedVersion, -} from './types.js'; +import type { MainModule, PgParserModule, SupportedVersion } from './types.js'; export type PgParserOptions = { version?: T; @@ -35,32 +30,20 @@ export class PgParser { return await createModule(); } - async parseSql(sql: string) { + async parse(sql: string) { const module = await this.#module; - const resultPtr = module.ccall('parse_sql', 'number', ['string'], [sql]); - const parseResult = await this.#parsePgQueryParseResult(resultPtr); - module.ccall('free_parse_result', undefined, ['number'], [resultPtr]); - - return parseResult; - } - - /** - * Parses a PgQueryParseResult struct from a pointer - */ - async #parsePgQueryParseResult(resultPtr: number): Promise> { - const module = await this.#module; - - const parseTreePtr = module.getValue(resultPtr, 'i32'); - const stderrBufferPtr = module.getValue(resultPtr + 4, 'i32'); - const errorPtr = module.getValue(resultPtr + 8, 'i32'); - - const tree = parseTreePtr - ? JSON.parse(module.UTF8ToString(parseTreePtr)) - : undefined; - const stderrBuffer = stderrBufferPtr - ? module.UTF8ToString(stderrBufferPtr) - : undefined; + const parseResultPtr = module.ccall( + 'parse_sql', + 'number', + ['string'], + [sql] + ); + + // Parse struct PgQueryProtobufParseResult from the pointer + const parseTreePtr = parseResultPtr; + const stderrBufferPtr = module.getValue(parseResultPtr + 8, 'i32'); + const errorPtr = module.getValue(parseResultPtr + 12, 'i32'); const error = errorPtr ? await this.#parsePgQueryError(errorPtr) : undefined; @@ -72,19 +55,66 @@ export class PgParser { }; } - if (!tree) { - throw new Error('both parse tree and error are undefined'); - } + const stderrBuffer = stderrBufferPtr + ? module.UTF8ToString(stderrBufferPtr) + : undefined; + + // Convert protobuf to JSON + const protobufToJsonResultPtr = module.ccall( + 'protobuf_to_json', + 'number', + ['number'], + [parseTreePtr] + ); + + const parseResult = await this.#parseProtobufToJsonResult( + protobufToJsonResultPtr + ); + + module.ccall('free_parse_result', undefined, ['number'], [parseResultPtr]); return { - tree, + tree: parseResult, error: undefined, stderrBuffer, }; } /** - * Parses a PgQueryError struct from a pointer + * Parses a ProtobufToJsonResult struct from a pointer. + */ + async #parseProtobufToJsonResult(resultPtr: number): Promise { + const module = await this.#module; + + const jsonStringPtr = module.getValue(resultPtr, 'i32'); + const errorPtr = module.getValue(resultPtr + 4, 'i32'); + + const jsonString = jsonStringPtr + ? module.UTF8ToString(jsonStringPtr) + : undefined; + const error = errorPtr ? module.UTF8ToString(errorPtr) : undefined; + + module.ccall( + 'free_protobuf_to_json_result', + undefined, + ['number'], + [resultPtr] + ); + + if (error) { + // This is unexpected, so throw instead of returning an error + throw new Error(error); + } + + if (!jsonString) { + throw new Error('both json string and error are undefined'); + } + + return JSON.parse(jsonString); + } + + /** + * Parses a PgQueryError struct from a pointer. */ async #parsePgQueryError(errorPtr: number) { const module = await this.#module; diff --git a/packages/pg-parser/tools/emsdk/Dockerfile b/packages/pg-parser/tools/emsdk/Dockerfile index ff63b0b..3fde14b 100644 --- a/packages/pg-parser/tools/emsdk/Dockerfile +++ b/packages/pg-parser/tools/emsdk/Dockerfile @@ -1,3 +1,8 @@ FROM emscripten/emsdk +RUN apt-get update && apt-get install -y \ + autoconf \ + automake \ + libtool + RUN npm i -g typescript tsx From a30807a3b01242ddb9adb8908e270558074b52ed Mon Sep 17 00:00:00 2001 From: Greg Richardson Date: Tue, 13 May 2025 14:32:44 -0600 Subject: [PATCH 2/2] wip: deparse --- packages/pg-parser/.clang-format | 4 + packages/pg-parser/Makefile | 33 +- packages/pg-parser/bindings/Filelists.mk | 3 +- packages/pg-parser/bindings/include/macros.h | 2 +- .../bindings/include/protobuf2json.h | 100 ++ packages/pg-parser/bindings/parse.c | 27 +- packages/pg-parser/bindings/protobuf-json.c | 114 ++- .../pg-parser/bindings/protobuf2json/base64.h | 121 +++ .../pg-parser/bindings/protobuf2json/bitmap.h | 36 + .../bindings/protobuf2json/protobuf2json.c | 858 ++++++++++++++++++ packages/pg-parser/src/index.ts | 2 +- packages/pg-parser/src/pg-parser.test.ts | 25 +- packages/pg-parser/src/pg-parser.ts | 126 ++- packages/pg-parser/src/types.ts | 12 + packages/pg-parser/src/util.ts | 18 +- packages/pg-parser/tools/emsdk/Dockerfile | 13 +- 16 files changed, 1420 insertions(+), 74 deletions(-) create mode 100644 packages/pg-parser/.clang-format create mode 100644 packages/pg-parser/bindings/include/protobuf2json.h create mode 100644 packages/pg-parser/bindings/protobuf2json/base64.h create mode 100644 packages/pg-parser/bindings/protobuf2json/bitmap.h create mode 100644 packages/pg-parser/bindings/protobuf2json/protobuf2json.c diff --git a/packages/pg-parser/.clang-format b/packages/pg-parser/.clang-format new file mode 100644 index 0000000..baae04e --- /dev/null +++ b/packages/pg-parser/.clang-format @@ -0,0 +1,4 @@ +BasedOnStyle: Google +IndentWidth: 2 +ColumnLimit: 0 +AllowShortIfStatementsOnASingleLine: false diff --git a/packages/pg-parser/Makefile b/packages/pg-parser/Makefile index 6a4d7e8..d407aa7 100644 --- a/packages/pg-parser/Makefile +++ b/packages/pg-parser/Makefile @@ -44,16 +44,6 @@ LIBPG_QUERY_PROTOBUF_C_DIR = $(LIBPG_QUERY_DIR)/vendor/protobuf-c LIBPG_QUERY_PROTOBUF_C_SRC_FILES = $(wildcard $(LIBPG_QUERY_PROTOBUF_C_DIR)/*.c) LIBPG_QUERY_PROTOBUF_C_OBJ_FILES = $(LIBPG_QUERY_PROTOBUF_C_SRC_FILES:.c=.o) -PROTOBUF_2_JSON_REPO = https://github.com/Sannis/protobuf2json-c.git -PROTOBUF_2_JSON_TAG = v0.4.0 -PROTOBUF_2_JSON_DIR = $(VENDOR_DIR)/protobuf2json-c/$(PROTOBUF_2_JSON_TAG) -PROTOBUF_2_JSON_SRC_DIR = $(PROTOBUF_2_JSON_DIR)/src -PROTOBUF_2_JSON_INCLUDE = $(PROTOBUF_2_JSON_DIR)/include -PROTOBUF_2_JSON_LIB = $(PROTOBUF_2_JSON_DIR)/protobuf2json.a -PROTOBUF_2_JSON_STAMP = $(PROTOBUF_2_JSON_DIR)/.stamp -PROTOBUF_2_JSON_FILES = $(wildcard $(PROTOBUF_2_JSON_SRC_DIR)/*.c) -PROTOBUF_2_JSON_OBJ_FILES = $(PROTOBUF_2_JSON_FILES:.c=.o) - JANSSON_REPO = https://github.com/akheron/jansson.git JANSSON_TAG = v2.14.1 JANSSON_DIR = $(VENDOR_DIR)/jansson/$(JANSSON_TAG) @@ -63,24 +53,17 @@ JANSSON_STAMP = $(JANSSON_DIR)/.stamp .DEFAULT_GOAL := build -$(OUTPUT_FILES): $(OBJ_FILES) $(LIBPG_QUERY_LIB) $(PROTOBUF_2_JSON_LIB) $(JANSSON_LIB) +$(OUTPUT_FILES): $(OBJ_FILES) $(LIBPG_QUERY_LIB) $(JANSSON_LIB) @mkdir -p $(OUTPUT_DIR) - $(CC) $(LDFLAGS) $(EMSCRIPTEN_FLAGS) -o $(OUTPUT_JS) $(OBJ_FILES) $(LIBPG_QUERY_LIB) $(JANSSON_LIB) $(PROTOBUF_2_JSON_LIB) --closure 0 --emit-tsd $(OUTPUT_D_TS) + $(CC) $(LDFLAGS) $(EMSCRIPTEN_FLAGS) -o $(OUTPUT_JS) $(OBJ_FILES) $(LIBPG_QUERY_LIB) $(JANSSON_LIB) --closure 0 --emit-tsd $(OUTPUT_D_TS) $(PROTOBUF_TYPE_GENERATOR) -i $(LIBPG_QUERY_DIR)/protobuf/pg_query.proto -o $(OUTPUT_DIR) -$(OBJ_FILES): %.o: %.c | $(LIBPG_QUERY_LIB) $(PROTOBUF_2_JSON_LIB) $(JANSSON_LIB) - $(CC) -I$(LIBPG_QUERY_DIR) -I$(LIBPG_QUERY_DIR)/vendor -I$(PROTOBUF_2_JSON_INCLUDE) -I$(JANSSON_SRC_DIR) -I$(INCLUDE) $(CFLAGS) -c $< -o $@ - -$(PROTOBUF_2_JSON_OBJ_FILES): %.o: %.c $(PROTOBUF_2_JSON_INCLUDE) $(JANSSON_LIB) - echo "Compiling $<" - $(CC) -I$(PROTOBUF_2_JSON_INCLUDE) -I$(JANSSON_SRC_DIR) $(CFLAGS) -c $< -o $@ +$(OBJ_FILES): %.o: %.c | $(LIBPG_QUERY_LIB) $(JANSSON_LIB) + $(CC) -I$(LIBPG_QUERY_DIR) -I$(LIBPG_QUERY_DIR)/vendor -I$(JANSSON_SRC_DIR) -I$(INCLUDE) $(CFLAGS) -c $< -o $@ $(LIBPG_QUERY_LIB): $(LIBPG_QUERY_STAMP) $(MAKE) -C $(LIBPG_QUERY_DIR) build -$(PROTOBUF_2_JSON_LIB): $(PROTOBUF_2_JSON_STAMP) $(PROTOBUF_2_JSON_OBJ_FILES) $(JANSSON_LIB) $(LIBPG_QUERY_PROTOBUF_C_OBJ_FILES) - $(AR) rcs $(PROTOBUF_2_JSON_LIB) $(PROTOBUF_2_JSON_OBJ_FILES) - $(JANSSON_LIB): $(JANSSON_STAMP) cd $(JANSSON_DIR) && \ $(AUTORECONF) -i && \ @@ -91,14 +74,6 @@ $(LIBPG_QUERY_STAMP): git clone -c advice.detachedHead=false --depth 1 --branch $(LIBPG_QUERY_TAG) $(LIBPG_QUERY_REPO) $(LIBPG_QUERY_DIR) touch $@ -$(PROTOBUF_2_JSON_STAMP): $(LIBPG_QUERY_STAMP) $(JANSSON_LIB) - git clone -c advice.detachedHead=false --depth 1 --branch $(PROTOBUF_2_JSON_TAG) $(PROTOBUF_2_JSON_REPO) $(PROTOBUF_2_JSON_DIR) - touch $@ - @mkdir -p $(PROTOBUF_2_JSON_INCLUDE)/google - ln -s ../../../../../$(LIBPG_QUERY_PROTOBUF_C_DIR) $(PROTOBUF_2_JSON_INCLUDE)/google/protobuf-c - ln -s ../../../../$(JANSSON_SRC_DIR)/jansson.h $(PROTOBUF_2_JSON_INCLUDE)/jansson.h - ln -s ../../../../$(JANSSON_SRC_DIR)/jansson_config.h $(PROTOBUF_2_JSON_INCLUDE)/jansson_config.h - $(JANSSON_STAMP): git clone -c advice.detachedHead=false --depth 1 --branch $(JANSSON_TAG) $(JANSSON_REPO) $(JANSSON_DIR) touch $@ diff --git a/packages/pg-parser/bindings/Filelists.mk b/packages/pg-parser/bindings/Filelists.mk index 205a797..ea48de2 100644 --- a/packages/pg-parser/bindings/Filelists.mk +++ b/packages/pg-parser/bindings/Filelists.mk @@ -1,3 +1,4 @@ SRC_FILES= \ $(SRC_DIR)/parse.c \ - $(SRC_DIR)/protobuf-json.c \ No newline at end of file + $(SRC_DIR)/protobuf-json.c \ + $(SRC_DIR)/protobuf2json/protobuf2json.c \ No newline at end of file diff --git a/packages/pg-parser/bindings/include/macros.h b/packages/pg-parser/bindings/include/macros.h index affa1c3..07708e3 100644 --- a/packages/pg-parser/bindings/include/macros.h +++ b/packages/pg-parser/bindings/include/macros.h @@ -3,4 +3,4 @@ #define EXPORT(name) __attribute__((export_name(name))) -#endif // MACROS_H \ No newline at end of file +#endif // MACROS_H \ No newline at end of file diff --git a/packages/pg-parser/bindings/include/protobuf2json.h b/packages/pg-parser/bindings/include/protobuf2json.h new file mode 100644 index 0000000..94db3e5 --- /dev/null +++ b/packages/pg-parser/bindings/include/protobuf2json.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2014-2016 Oleg Efimov + * + * protobuf2json-c is free software; you can redistribute it + * and/or modify it under the terms of the MIT license. + * See LICENSE for details. + */ + +#ifndef PROTOBUF2JSON_H +#define PROTOBUF2JSON_H 1 + +#include +#include + +/* Common errors */ +#define PROTOBUF2JSON_ERR_CANNOT_ALLOCATE_MEMORY -001 +#define PROTOBUF2JSON_ERR_UNSUPPORTED_FIELD_TYPE -002 +#define PROTOBUF2JSON_ERR_UNKNOWN_ENUM_VALUE -003 + +/* protobuf2json_string */ +#define PROTOBUF2JSON_ERR_CANNOT_DUMP_STRING -101 +/* protobuf2json_file */ +#define PROTOBUF2JSON_ERR_CANNOT_DUMP_FILE -102 +/* protobuf2json */ +#define PROTOBUF2JSON_ERR_JANSSON_INTERNAL -201 + +/*json2protobuf_string*/ +#define PROTOBUF2JSON_ERR_CANNOT_PARSE_STRING -301 +/* json2protobuf_file */ +#define PROTOBUF2JSON_ERR_CANNOT_PARSE_FILE -302 +/* json2protobuf */ +#define PROTOBUF2JSON_ERR_UNKNOWN_FIELD -401 +#define PROTOBUF2JSON_ERR_IS_NOT_OBJECT -402 +#define PROTOBUF2JSON_ERR_IS_NOT_ARRAY -403 +#define PROTOBUF2JSON_ERR_IS_NOT_INTEGER -404 +#define PROTOBUF2JSON_ERR_IS_NOT_INTEGER_OR_REAL -405 +#define PROTOBUF2JSON_ERR_IS_NOT_BOOLEAN -406 +#define PROTOBUF2JSON_ERR_IS_NOT_STRING -407 +#define PROTOBUF2JSON_ERR_REQUIRED_IS_MISSING -408 +/*#define PROTOBUF2JSON_ERR_DUPLICATE_FIELD -???*/ + +#ifdef __cplusplus +extern "C" { +#endif + +/* === Protobuf -> JSON === */ + +int protobuf2json_object( + ProtobufCMessage *protobuf_message, + json_t **json_object, + char *error_string, + size_t error_size); + +int protobuf2json_string( + ProtobufCMessage *protobuf_message, + size_t json_flags, + char **json_string, + char *error_string, + size_t error_size); + +int protobuf2json_file( + ProtobufCMessage *protobuf_message, + size_t json_flags, + char *json_file, + char *fopen_mode, + char *error_string, + size_t error_size); + +/* === JSON -> Protobuf === */ + +int json2protobuf_object( + json_t *json_object, + const ProtobufCMessageDescriptor *protobuf_message_descriptor, + ProtobufCMessage **protobuf_message, + char *error_string, + size_t error_size); + +int json2protobuf_string( + char *json_string, + size_t json_flags, + const ProtobufCMessageDescriptor *protobuf_message_descriptor, + ProtobufCMessage **protobuf_message, + char *error_string, + size_t error_size); + +int json2protobuf_file( + char *json_file, + size_t json_flags, + const ProtobufCMessageDescriptor *protobuf_message_descriptor, + ProtobufCMessage **protobuf_message, + char *error_string, + size_t error_size); + +/* === END === */ + +#ifdef __cplusplus +} +#endif + +#endif /* PROTOBUF2JSON_H */ diff --git a/packages/pg-parser/bindings/parse.c b/packages/pg-parser/bindings/parse.c index 26238b2..49388ba 100644 --- a/packages/pg-parser/bindings/parse.c +++ b/packages/pg-parser/bindings/parse.c @@ -1,18 +1,35 @@ +#include #include -#include "pg_query.h" + #include "macros.h" +#include "pg_query.h" EXPORT("parse_sql") -PgQueryProtobufParseResult *parse_sql(char *sql) -{ +PgQueryProtobufParseResult *parse_sql(char *sql) { PgQueryProtobufParseResult *result = (PgQueryProtobufParseResult *)malloc(sizeof(PgQueryProtobufParseResult)); *result = pg_query_parse_protobuf(sql); return result; } +EXPORT("deparse_sql") +PgQueryDeparseResult *deparse_sql(PgQueryProtobuf *parse_tree) { + PgQueryDeparseResult *result = (PgQueryDeparseResult *)malloc(sizeof(PgQueryDeparseResult)); + printf("Deparse parse tree: %p\n", parse_tree); + printf("Deparse parse tree length: %d\n", parse_tree->len); + *result = pg_query_deparse_protobuf(*parse_tree); + printf("Deparse error: %p\n", result->error); + printf("Deparse result: %s\n", result->query); + return result; +} + EXPORT("free_parse_result") -void free_parse_result(PgQueryProtobufParseResult *result) -{ +void free_parse_result(PgQueryProtobufParseResult *result) { pg_query_free_protobuf_parse_result(*result); free(result); +} + +EXPORT("free_deparse_result") +void free_deparse_result(PgQueryDeparseResult *result) { + pg_query_free_deparse_result(*result); + free(result); } \ No newline at end of file diff --git a/packages/pg-parser/bindings/protobuf-json.c b/packages/pg-parser/bindings/protobuf-json.c index 6efdd28..a9608b1 100644 --- a/packages/pg-parser/bindings/protobuf-json.c +++ b/packages/pg-parser/bindings/protobuf-json.c @@ -1,63 +1,137 @@ #define _POSIX_C_SOURCE 200809L -#include +#include #include +#include #include +#include "macros.h" #include "pg_query.h" -#include "protobuf2json.h" #include "protobuf/pg_query.pb-c.h" -#include "macros.h" +#include "protobuf2json.h" -typedef struct -{ +typedef struct { char *json_string; char *error; } ProtobufToJsonResult; +typedef struct { + PgQueryProtobuf protobuf; + char *error; +} JsonToProtobufResult; + EXPORT("protobuf_to_json") -ProtobufToJsonResult *protobuf_to_json(PgQueryProtobuf *protobuf) -{ +ProtobufToJsonResult *protobuf_to_json(PgQueryProtobuf *protobuf) { ProtobufToJsonResult *result = (ProtobufToJsonResult *)malloc(sizeof(ProtobufToJsonResult)); // Unpack the protobuf binary data back into a message PgQuery__ParseResult *parse_result = pg_query__parse_result__unpack(NULL, protobuf->len, (uint8_t *)protobuf->data); - if (!parse_result) - { + if (!parse_result) { result->error = strdup("Failed to unpack protobuf message"); return result; } + printf("size of protobuf: %zu\n", pg_query__parse_result__get_packed_size(parse_result)); - char *error = (char *)malloc(256); + char *error = (char *)malloc(sizeof(char) * 256); - // Convert the protobuf message to JSON + // Convert the protobuf message to a JSON string int ret = protobuf2json_string( &parse_result->base, 0, &result->json_string, error, - sizeof(*error)); + 256); + + free(parse_result); + + if (ret != 0) { + result->error = error; + return result; + } + + free(error); + return result; +} + +EXPORT("json_to_protobuf") +JsonToProtobufResult *json_to_protobuf(char *json_string) { + JsonToProtobufResult *result = (JsonToProtobufResult *)malloc(sizeof(JsonToProtobufResult)); + ProtobufCMessage *protobuf_message = NULL; + + char *error = (char *)malloc(sizeof(char) * 256); + + printf("json_string: %s\n", json_string); + + // Convert the JSON string to a protobuf message + int ret = json2protobuf_string( + json_string, + 0, + &pg_query__parse_result__descriptor, + &protobuf_message, + error, + 256); - if (ret != 0) - { + if (ret != 0) { result->error = error; return result; } free(error); + + PgQuery__ParseResult *parse_result = (PgQuery__ParseResult *)protobuf_message; + + printf("size of protobuf: %zu\n", pg_query__parse_result__get_packed_size(parse_result)); + + char *error2 = (char *)malloc(sizeof(char) * 256); + char *json_string_reparsed = NULL; + + // Convert the protobuf message to a JSON string + protobuf2json_string( + &parse_result->base, + 0, + &json_string_reparsed, + error2, + 256); + + printf("json_string after reparse: %s\n", json_string_reparsed); + + result->protobuf.len = pg_query__parse_result__get_packed_size(parse_result); + result->protobuf.data = malloc(sizeof(char) * result->protobuf.len); + // result->protobuf.len = protobuf_c_message_get_packed_size(protobuf_message); + // result->protobuf.data = malloc(sizeof(char) * result->protobuf.len); + + // Pack the protobuf message into binary data + pg_query__parse_result__pack(parse_result, (uint8_t *)result->protobuf.data); + // protobuf_c_message_pack(protobuf_message, (uint8_t *)result->protobuf.data); + + printf("pointer of result: %p\n", result); + printf("pointer of result->protobuf: %p\n", &result->protobuf); + printf("result->protobuf.len: %d\n", result->protobuf.len); + + // Free the protobuf message since we've packed it + // protobuf_c_message_free_unpacked((ProtobufCMessage *)protobuf_message, NULL); + return result; } EXPORT("free_protobuf_to_json_result") -void free_protobuf_to_json_result(ProtobufToJsonResult *result) -{ - if (result->json_string) - { +void free_protobuf_to_json_result(ProtobufToJsonResult *result) { + if (result->json_string) { free(result->json_string); } - if (result->error) - { + if (result->error) { free(result->error); } free(result); } + +EXPORT("free_json_to_protobuf_result") +void free_json_to_protobuf_result(JsonToProtobufResult *result) { + if (result->protobuf.data) { + free(result->protobuf.data); + } + if (result->error) { + free(result->error); + } + free(result); +} \ No newline at end of file diff --git a/packages/pg-parser/bindings/protobuf2json/base64.h b/packages/pg-parser/bindings/protobuf2json/base64.h new file mode 100644 index 0000000..f36ada5 --- /dev/null +++ b/packages/pg-parser/bindings/protobuf2json/base64.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2014-2016 Oleg Efimov + * + * Code based on nginx 1.5.5, extracted by Anton Povarov + * + * protobuf2json-c is free software; you can redistribute it + * and/or modify it under the terms of the MIT license. + * See LICENSE for details. + */ + +#ifndef BASE64_H +#define BASE64_H 1 + +#define base64_encoded_len(len) (((len + 2) / 3) * 4) +#define base64_decoded_len(len) (((len + 3) / 4) * 3) + +typedef struct base64_tables { + unsigned char encode[64]; + unsigned char decode[256]; + unsigned char padding; +} base64_tables_t; + +static const base64_tables_t base64_default_tables = { + .encode = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", + .decode = { + 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 62, 77, 77, 77, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 77, 77, 77, 77, 77, 77, + 77, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 77, 77, 77, 77, 77, + 77, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 77, 77, 77, 77, 77, + + 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, + 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77, 77 + }, + .padding = '=' +}; + +size_t base64_encode(char *dst, const char *src, size_t src_len) +{ + char *d = dst; + const unsigned char *s = (void*)src; + const unsigned char *basis64 = base64_default_tables.encode; + + while (src_len > 2) { + *d++ = basis64[(s[0] >> 2) & 0x3f]; + *d++ = basis64[((s[0] & 3) << 4) | (s[1] >> 4)]; + *d++ = basis64[((s[1] & 0x0f) << 2) | (s[2] >> 6)]; + *d++ = basis64[s[2] & 0x3f]; + + s += 3; + src_len -= 3; + } + + if (src_len) { + *d++ = basis64[(s[0] >> 2) & 0x3f]; + + if (src_len == 1) { + *d++ = basis64[(s[0] & 3) << 4]; + *d++ = base64_default_tables.padding; + } else { + *d++ = basis64[((s[0] & 3) << 4) | (s[1] >> 4)]; + *d++ = basis64[(s[1] & 0x0f) << 2]; + } + + *d++ = base64_default_tables.padding; + } + + return (d - dst); +} + +size_t base64_decode(char *dst, const char *src, size_t src_len) +{ + size_t len; + char *d = dst; + const unsigned char *s = (void*)src; + const unsigned char *basis = base64_default_tables.decode; + + for (len = 0; len < src_len; len++) { + if (s[len] == base64_default_tables.padding) { + break; + } + + if (basis[s[len]] == 77) { + return 0; + } + } + + if (len % 4 == 1) { + return 0; + } + + while (len > 3) { + *d++ = (char) (basis[s[0]] << 2 | basis[s[1]] >> 4); + *d++ = (char) (basis[s[1]] << 4 | basis[s[2]] >> 2); + *d++ = (char) (basis[s[2]] << 6 | basis[s[3]]); + + s += 4; + len -= 4; + } + + if (len > 1) { + *d++ = (char) (basis[s[0]] << 2 | basis[s[1]] >> 4); + } + + if (len > 2) { + *d++ = (char) (basis[s[1]] << 4 | basis[s[2]] >> 2); + } + + return (d - dst); +} + +#endif /* BASE64_H */ diff --git a/packages/pg-parser/bindings/protobuf2json/bitmap.h b/packages/pg-parser/bindings/protobuf2json/bitmap.h new file mode 100644 index 0000000..af18648 --- /dev/null +++ b/packages/pg-parser/bindings/protobuf2json/bitmap.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2014-2016 Oleg Efimov + * + * protobuf2json-c is free software; you can redistribute it + * and/or modify it under the terms of the MIT license. + * See LICENSE for details. + */ + +#ifndef BITMAP_H +#define BITMAP_H 1 + +typedef unsigned char bitmap_word_t; +typedef bitmap_word_t* bitmap_t; + +#define bitmap_word_t_bits (8 * sizeof(bitmap_word_t)) +#define bitmap_words_needed(size) (((size) + (bitmap_word_t_bits - 1)) / bitmap_word_t_bits) + +bitmap_t bitmap_alloc(int size) +{ + return (bitmap_t)calloc(bitmap_words_needed(size), sizeof(bitmap_word_t)); +} + +void bitmap_free(bitmap_t bitmap) +{ + free(bitmap); +} + +void bitmap_set(bitmap_t bitmap, unsigned int i) { + bitmap[i / bitmap_word_t_bits] |= (1 << (i & (bitmap_word_t_bits - 1))); +} + +int bitmap_get(bitmap_t bitmap, unsigned int i) { + return (bitmap[i / bitmap_word_t_bits] & (1 << (i & (bitmap_word_t_bits - 1)))) ? 1 : 0; +} + +#endif /* BITMAP_H */ diff --git a/packages/pg-parser/bindings/protobuf2json/protobuf2json.c b/packages/pg-parser/bindings/protobuf2json/protobuf2json.c new file mode 100644 index 0000000..96a91b9 --- /dev/null +++ b/packages/pg-parser/bindings/protobuf2json/protobuf2json.c @@ -0,0 +1,858 @@ +/* + * Adapted from https://github.com/Sannis/protobuf2json-c + * + * --- + * + * Copyright (c) 2014-2016 Oleg Efimov + * + * protobuf2json-c is free software; you can redistribute it + * and/or modify it under the terms of the MIT license. + * See LICENSE for details. + */ + +#include +#include +#include +#include + +/* Interface definitions */ +#include "protobuf2json.h" + +/* Simple bitmap implementation */ +#include "bitmap.h" + +/* Simple base64 implementation */ +#include "base64.h" + +/* === Defines === obviously private === */ + +#define SET_ERROR_STRING_AND_RETURN(error, error_string_format, ...) \ + do { \ + if (error_string && error_size) { \ + snprintf( \ + error_string, error_size, \ + error_string_format, \ + ##__VA_ARGS__); \ + } \ + return error; \ + } while (0) + +/* === Protobuf -> JSON === Private === */ + +const ProtobufCFieldDescriptor * +protobuf_c_message_descriptor_get_field_by_json_name(const ProtobufCMessageDescriptor *desc, + const char *json_name) { + if (desc == NULL) + return NULL; + + // Linear search through all fields since we can't use binary search on json_name + for (unsigned i = 0; i < desc->n_fields; i++) { + const ProtobufCFieldDescriptor *field = &desc->fields[i]; + if (strcmp(field->reserved2, json_name) == 0) { + return field; + } + } + + return NULL; +} + +static size_t protobuf2json_value_size_by_type(ProtobufCType type) { + switch (type) { + case PROTOBUF_C_TYPE_INT32: + case PROTOBUF_C_TYPE_SINT32: + case PROTOBUF_C_TYPE_SFIXED32: + case PROTOBUF_C_TYPE_UINT32: + case PROTOBUF_C_TYPE_FIXED32: + return 4; + case PROTOBUF_C_TYPE_INT64: + case PROTOBUF_C_TYPE_SINT64: + case PROTOBUF_C_TYPE_SFIXED64: + case PROTOBUF_C_TYPE_UINT64: + case PROTOBUF_C_TYPE_FIXED64: + return 8; + case PROTOBUF_C_TYPE_FLOAT: + return 4; + case PROTOBUF_C_TYPE_DOUBLE: + return 8; + case PROTOBUF_C_TYPE_BOOL: + return sizeof(protobuf_c_boolean); + case PROTOBUF_C_TYPE_ENUM: + return 4; + case PROTOBUF_C_TYPE_STRING: + return sizeof(char *); + case PROTOBUF_C_TYPE_BYTES: + return sizeof(ProtobufCBinaryData); + case PROTOBUF_C_TYPE_MESSAGE: + return sizeof(ProtobufCMessage *); + default: + assert(0); + return 0; + } +} + +static int protobuf2json_process_message( + const ProtobufCMessage *protobuf_message, + json_t **json_message, + char *error_string, + size_t error_size); + +static int protobuf2json_process_field( + const ProtobufCFieldDescriptor *field_descriptor, + const void *protobuf_value, + json_t **json_value, + char *error_string, + size_t error_size) { + switch (field_descriptor->type) { + case PROTOBUF_C_TYPE_INT32: + case PROTOBUF_C_TYPE_SINT32: + case PROTOBUF_C_TYPE_SFIXED32: + *json_value = json_integer(*(int32_t *)protobuf_value); + break; + case PROTOBUF_C_TYPE_UINT32: + case PROTOBUF_C_TYPE_FIXED32: + *json_value = json_integer(*(uint32_t *)protobuf_value); + break; + case PROTOBUF_C_TYPE_INT64: + case PROTOBUF_C_TYPE_SINT64: + case PROTOBUF_C_TYPE_SFIXED64: + *json_value = json_integer(*(int64_t *)protobuf_value); + break; + case PROTOBUF_C_TYPE_UINT64: + case PROTOBUF_C_TYPE_FIXED64: + *json_value = json_integer(*(uint64_t *)protobuf_value); + break; + case PROTOBUF_C_TYPE_FLOAT: + *json_value = json_real(*(float *)protobuf_value); + break; + case PROTOBUF_C_TYPE_DOUBLE: + *json_value = json_real(*(double *)protobuf_value); + break; + case PROTOBUF_C_TYPE_BOOL: + *json_value = json_boolean(*(protobuf_c_boolean *)protobuf_value); + break; + case PROTOBUF_C_TYPE_ENUM: { + const ProtobufCEnumValue *protobuf_enum_value = protobuf_c_enum_descriptor_get_value( + field_descriptor->descriptor, + *(int *)protobuf_value); + + if (!protobuf_enum_value) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_UNKNOWN_ENUM_VALUE, + "Unknown value %d for enum '%s'", + *(int *)protobuf_value, ((ProtobufCEnumDescriptor *)field_descriptor->descriptor)->name); + } + + *json_value = json_string((char *)protobuf_enum_value->name); + + break; + } + case PROTOBUF_C_TYPE_STRING: + *json_value = json_string(*(char **)protobuf_value); + break; + case PROTOBUF_C_TYPE_BYTES: { + const ProtobufCBinaryData *protobuf_binary = (const ProtobufCBinaryData *)protobuf_value; + + int base64_encoded_length = base64_encoded_len(protobuf_binary->len); + + char *base64_encoded_data = calloc(base64_encoded_length, sizeof(char)); + if (!base64_encoded_data) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_ALLOCATE_MEMORY, + "Cannot allocate %zu bytes using calloc(3)", + base64_encoded_length * sizeof(char)); + } + + base64_encoded_length = base64_encode(base64_encoded_data, (const char *)protobuf_binary->data, protobuf_binary->len); + + *json_value = json_stringn((const char *)base64_encoded_data, base64_encoded_length); + + free(base64_encoded_data); + + break; + } + case PROTOBUF_C_TYPE_MESSAGE: { + const ProtobufCMessage **protobuf_message = (const ProtobufCMessage **)protobuf_value; + + int result = protobuf2json_process_message(*protobuf_message, json_value, error_string, error_size); + if (result) { + return result; + } + + break; + } + default: + assert(0); + } + + if (!*json_value) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_ALLOCATE_MEMORY, + "Cannot allocate JSON structure in protobuf2json_process_field()"); + } + + return 0; +} + +static int protobuf2json_process_message( + const ProtobufCMessage *protobuf_message, + json_t **json_message, + char *error_string, + size_t error_size) { + *json_message = json_object(); + if (!*json_message) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_ALLOCATE_MEMORY, + "Cannot allocate JSON structure using json_object()"); + } + + json_t *json_value = NULL; + + unsigned i; + for (i = 0; i < protobuf_message->descriptor->n_fields; i++) { + const ProtobufCFieldDescriptor *field_descriptor = protobuf_message->descriptor->fields + i; + const void *protobuf_value = ((const char *)protobuf_message) + field_descriptor->offset; + const void *protobuf_value_quantifier = ((const char *)protobuf_message) + field_descriptor->quantifier_offset; + + if (field_descriptor->label == PROTOBUF_C_LABEL_REQUIRED) { + json_value = NULL; + + int result = protobuf2json_process_field(field_descriptor, protobuf_value, &json_value, error_string, error_size); + if (result) { + return result; + } + + if (json_object_set_new(*json_message, field_descriptor->reserved2, json_value)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_JANSSON_INTERNAL, + "Error in json_object_set_new()"); + } + } else if (field_descriptor->label == PROTOBUF_C_LABEL_OPTIONAL || field_descriptor->label == PROTOBUF_C_LABEL_NONE) { + if (field_descriptor->flags & PROTOBUF_C_FIELD_FLAG_ONEOF) { + if (*(uint32_t *)protobuf_value_quantifier == field_descriptor->id) { + if (field_descriptor->type == PROTOBUF_C_TYPE_MESSAGE || field_descriptor->type == PROTOBUF_C_TYPE_STRING) { + if (protobuf_value == NULL || protobuf_value == field_descriptor->default_value) { + continue; + } + } + } else { + continue; + } + } + + protobuf_c_boolean is_set = 0; + + if (field_descriptor->type == PROTOBUF_C_TYPE_MESSAGE || field_descriptor->type == PROTOBUF_C_TYPE_STRING) { + if (*(const void *const *)protobuf_value) { + is_set = 1; + } + } else { + if (*(const protobuf_c_boolean *)protobuf_value_quantifier) { + is_set = 1; + } + } + + if (is_set || field_descriptor->default_value) { + json_value = NULL; + + int result = protobuf2json_process_field(field_descriptor, protobuf_value, &json_value, error_string, error_size); + if (result) { + return result; + } + + if (json_object_set_new(*json_message, field_descriptor->reserved2, json_value)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_JANSSON_INTERNAL, + "Error in json_object_set_new()"); + } + } + } else if (field_descriptor->label == PROTOBUF_C_LABEL_REPEATED) { // PROTOBUF_C_LABEL_REPEATED + const size_t *protobuf_values_count = (const size_t *)protobuf_value_quantifier; + + if (*protobuf_values_count) { + json_t *array = json_array(); + if (!array) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_ALLOCATE_MEMORY, + "Cannot allocate JSON structure using json_array()"); + } + + size_t value_size = protobuf2json_value_size_by_type(field_descriptor->type); + if (!value_size) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_UNSUPPORTED_FIELD_TYPE, + "Cannot calculate value size for %d using protobuf2json_value_size_by_type()", + field_descriptor->type); + } + + unsigned j; + for (j = 0; j < *protobuf_values_count; j++) { + const char *protobuf_value_repeated = (*(char *const *)protobuf_value) + j * value_size; + + json_value = NULL; + + int result = protobuf2json_process_field(field_descriptor, (const void *)protobuf_value_repeated, &json_value, error_string, error_size); + if (result) { + return result; + } + + if (json_array_append_new(array, json_value)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_JANSSON_INTERNAL, + "Error in json_array_append_new()"); + } + } + + if (json_object_set_new(*json_message, field_descriptor->reserved2, array)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_JANSSON_INTERNAL, + "Error in json_object_set_new()"); + } + } + } else { + // error, this should not happen + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_UNSUPPORTED_FIELD_TYPE, + "Unsupported field label %d for field '%s' in message '%s'", + field_descriptor->label, field_descriptor->name, protobuf_message->descriptor->name); + } + } + + return 0; +} + +/* === Protobuf -> JSON === Public === */ + +int protobuf2json_object( + ProtobufCMessage *protobuf_message, + json_t **json_object, + char *error_string, + size_t error_size) { + int ret = protobuf2json_process_message(protobuf_message, json_object, error_string, error_size); + if (ret) { + json_decref(*json_object); + return ret; + } + + return 0; +} + +int protobuf2json_string( + ProtobufCMessage *protobuf_message, + size_t json_flags, + char **json_string, + char *error_string, + size_t error_size) { + json_t *json_object = NULL; + + int ret = protobuf2json_object(protobuf_message, &json_object, error_string, error_size); + if (ret) { + return ret; + } + + // NOTICE: Should be freed by caller + *json_string = json_dumps(json_object, json_flags); + if (!*json_string) { + json_decref(json_object); + + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_DUMP_STRING, + "Cannot dump JSON object to string using json_dumps()"); + } + + json_decref(json_object); + return 0; +} + +/* === JSON -> Protobuf === Private === */ + +static int json2protobuf_process_message( + json_t *json_object, + const ProtobufCMessageDescriptor *protobuf_message_descriptor, + ProtobufCMessage **protobuf_message, + char *error_string, + size_t error_size); + +static const char *json2protobuf_integer_name_by_c_type(ProtobufCType type) { + switch (type) { + case PROTOBUF_C_TYPE_INT32: + return "int32"; + case PROTOBUF_C_TYPE_SINT32: + return "sint32"; + case PROTOBUF_C_TYPE_SFIXED32: + return "sfixed32"; + case PROTOBUF_C_TYPE_UINT32: + return "uint32"; + case PROTOBUF_C_TYPE_FIXED32: + return "fixed32"; + case PROTOBUF_C_TYPE_INT64: + return "int64"; + case PROTOBUF_C_TYPE_SINT64: + return "sint64"; + case PROTOBUF_C_TYPE_SFIXED64: + return "sfixed64"; + case PROTOBUF_C_TYPE_UINT64: + return "uint64"; + case PROTOBUF_C_TYPE_FIXED64: + return "fixed64"; + default: + assert(0); + return "unknown"; + } +} + +static int json2protobuf_process_field( + const ProtobufCFieldDescriptor *field_descriptor, + json_t *json_value, + void *protobuf_value, + char *error_string, + size_t error_size) { + printf("json2protobuf_process_field: %s\n", field_descriptor->name); + + if (field_descriptor->type == PROTOBUF_C_TYPE_INT32 || field_descriptor->type == PROTOBUF_C_TYPE_SINT32 || field_descriptor->type == PROTOBUF_C_TYPE_SFIXED32) { + if (!json_is_integer(json_value)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_INTEGER, + "JSON value is not an integer required for GPB %s", + json2protobuf_integer_name_by_c_type(field_descriptor->type)); + } + + int32_t value_int32_t = (int32_t)json_integer_value(json_value); + + memcpy(protobuf_value, &value_int32_t, sizeof(value_int32_t)); + } else if (field_descriptor->type == PROTOBUF_C_TYPE_UINT32 || field_descriptor->type == PROTOBUF_C_TYPE_FIXED32) { + if (!json_is_integer(json_value)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_INTEGER, + "JSON value is not an integer required for GPB %s", + json2protobuf_integer_name_by_c_type(field_descriptor->type)); + } + + uint32_t value_uint32_t = (uint32_t)json_integer_value(json_value); + + memcpy(protobuf_value, &value_uint32_t, sizeof(value_uint32_t)); + } else if (field_descriptor->type == PROTOBUF_C_TYPE_INT64 || field_descriptor->type == PROTOBUF_C_TYPE_SINT64 || field_descriptor->type == PROTOBUF_C_TYPE_SFIXED64) { + if (!json_is_integer(json_value)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_INTEGER, + "JSON value is not an integer required for GPB %s", + json2protobuf_integer_name_by_c_type(field_descriptor->type)); + } + + int64_t value_int64_t = (int64_t)json_integer_value(json_value); + + memcpy(protobuf_value, &value_int64_t, sizeof(value_int64_t)); + } else if (field_descriptor->type == PROTOBUF_C_TYPE_UINT64 || field_descriptor->type == PROTOBUF_C_TYPE_FIXED64) { + if (!json_is_integer(json_value)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_INTEGER, + "JSON value is not an integer required for GPB %s", + json2protobuf_integer_name_by_c_type(field_descriptor->type)); + } + + uint64_t value_uint64_t = (uint64_t)json_integer_value(json_value); + + memcpy(protobuf_value, &value_uint64_t, sizeof(value_uint64_t)); + } else if (field_descriptor->type == PROTOBUF_C_TYPE_FLOAT) { + float value_float; + + if (json_is_integer(json_value)) { + value_float = (float)json_integer_value(json_value); + } else if (json_is_real(json_value)) { + value_float = (float)json_real_value(json_value); + } else { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_INTEGER_OR_REAL, + "JSON value is not a integer/real required for GPB float"); + } + + memcpy(protobuf_value, &value_float, sizeof(value_float)); + } else if (field_descriptor->type == PROTOBUF_C_TYPE_DOUBLE) { + double value_double; + + if (json_is_integer(json_value)) { + value_double = (double)json_integer_value(json_value); + } else if (json_is_real(json_value)) { + value_double = (double)json_real_value(json_value); + } else { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_INTEGER_OR_REAL, + "JSON value is not a integer/real required for GPB double"); + } + + memcpy(protobuf_value, &value_double, sizeof(value_double)); + } else if (field_descriptor->type == PROTOBUF_C_TYPE_BOOL) { + if (!json_is_boolean(json_value)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_BOOLEAN, + "JSON value is not a boolean required for GPB bool"); + } + + protobuf_c_boolean value_boolean = (protobuf_c_boolean)json_boolean_value(json_value); + + memcpy(protobuf_value, &value_boolean, sizeof(value_boolean)); + } else if (field_descriptor->type == PROTOBUF_C_TYPE_ENUM) { + if (!json_is_string(json_value)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_STRING, + "JSON value is not a string required for GPB enum"); + } + + const char *enum_value_name = json_string_value(json_value); + + const ProtobufCEnumValue *enum_value; + + enum_value = protobuf_c_enum_descriptor_get_value_by_name(field_descriptor->descriptor, enum_value_name); + if (!enum_value) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_UNKNOWN_ENUM_VALUE, + "Unknown value '%s' for enum '%s'", + enum_value_name, ((ProtobufCEnumDescriptor *)field_descriptor->descriptor)->name); + } + + int32_t value_enum = (int32_t)enum_value->value; + + memcpy(protobuf_value, &value_enum, sizeof(value_enum)); + } else if (field_descriptor->type == PROTOBUF_C_TYPE_STRING) { + if (!json_is_string(json_value)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_STRING, + "JSON value is not a string required for GPB string"); + } + + const char *value_string = json_string_value(json_value); + size_t value_string_length = strlen(value_string); + + char *value_string_copy = calloc(value_string_length + 1, sizeof(char)); + if (!value_string_copy) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_ALLOCATE_MEMORY, + "Cannot allocate %zu bytes using calloc(3)", + (value_string_length + 1) * sizeof(char)); + } + + memcpy(value_string_copy, value_string, value_string_length + 1); + + *(char **)(protobuf_value) = value_string_copy; + } else if (field_descriptor->type == PROTOBUF_C_TYPE_BYTES) { + if (!json_is_string(json_value)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_STRING, + "JSON value is not a string required for GPB bytes"); + } + + const char *value_string = json_string_value(json_value); + size_t value_string_length = json_string_length(json_value); + + int base64_decoded_length = base64_decoded_len(value_string_length); + + char *base64_decoded_data = calloc(base64_decoded_length, sizeof(char)); + if (!base64_decoded_data) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_ALLOCATE_MEMORY, + "Cannot allocate %zu bytes using calloc(3)", + base64_decoded_length * sizeof(char)); + } + + /* @todo: check for zero length / error */ + base64_decoded_length = base64_decode(base64_decoded_data, value_string, value_string_length); + + char *value_string_copy = calloc(base64_decoded_length, sizeof(char)); + if (!value_string_copy) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_ALLOCATE_MEMORY, + "Cannot allocate %zu bytes using calloc(3)", + base64_decoded_length * sizeof(char)); + } + + memcpy(value_string_copy, base64_decoded_data, base64_decoded_length); + + free(base64_decoded_data); + + ProtobufCBinaryData value_binary; + + value_binary.data = (uint8_t *)value_string_copy; + value_binary.len = base64_decoded_length; + + memcpy(protobuf_value, &value_binary, sizeof(value_binary)); + } else if (field_descriptor->type == PROTOBUF_C_TYPE_MESSAGE) { + ProtobufCMessage *protobuf_message = NULL; + + printf("json2protobuf_process_message: %s\n", field_descriptor->name); + + int result = json2protobuf_process_message(json_value, field_descriptor->descriptor, &protobuf_message, error_string, error_size); + if (result) { + return result; + } + + memcpy(protobuf_value, &protobuf_message, sizeof(protobuf_message)); + } else { + assert(0); + } + + return 0; +} + +#define SAFE_FREE_BITMAP_AND_MESSAGE \ + do { \ + if (presented_fields) { \ + bitmap_free(presented_fields); \ + } \ + if (protobuf_message) { \ + protobuf_c_message_free_unpacked(*protobuf_message, NULL); \ + *protobuf_message = NULL; \ + } \ + } while (0) + +static int json2protobuf_process_message( + json_t *json_object, + const ProtobufCMessageDescriptor *protobuf_message_descriptor, + ProtobufCMessage **protobuf_message, + char *error_string, + size_t error_size) { + bitmap_t presented_fields = NULL; + + int result = 0; + + if (!json_is_object(json_object)) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_OBJECT, + "JSON is not an object required for GPB message"); + } + + printf("allocating %zu bytes for GPB message '%s'\n", + protobuf_message_descriptor->sizeof_message, protobuf_message_descriptor->name); + + *protobuf_message = calloc(1, protobuf_message_descriptor->sizeof_message); + if (!*protobuf_message) { + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_ALLOCATE_MEMORY, + "Cannot allocate %zu bytes using calloc(3)", + protobuf_message_descriptor->sizeof_message); + } + + protobuf_c_message_init(protobuf_message_descriptor, *protobuf_message); + + presented_fields = bitmap_alloc(protobuf_message_descriptor->n_fields); + if (!presented_fields) { + SAFE_FREE_BITMAP_AND_MESSAGE; + + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_ALLOCATE_MEMORY, + "Cannot allocate bitmap structure using bitmap_alloc()"); + } + + const char *json_key; + json_t *json_object_value; + json_object_foreach(json_object, json_key, json_object_value) { + const ProtobufCFieldDescriptor *field_descriptor = protobuf_c_message_descriptor_get_field_by_json_name(protobuf_message_descriptor, json_key); + + if (!field_descriptor) { + SAFE_FREE_BITMAP_AND_MESSAGE; + + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_UNKNOWN_FIELD, + "Unknown field '%s' for message '%s'", + json_key, protobuf_message_descriptor->name); + } + + unsigned int field_number = field_descriptor - protobuf_message_descriptor->fields; + + // This cannot happen because Jansson handle this on his side + /*if (bitmap_get(presented_fields, field_number)) { + SAFE_FREE_BITMAP_AND_MESSAGE; + + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_DUPLICATE_FIELD, + "Duplicate field '%s' for message '%s'", + json_key, protobuf_message_descriptor->name + ); + }*/ + bitmap_set(presented_fields, field_number); + + void *protobuf_value = ((char *)*protobuf_message) + field_descriptor->offset; + void *protobuf_value_quantifier = ((char *)*protobuf_message) + field_descriptor->quantifier_offset; + + printf("json_key: %s\n", json_key); + printf("field_descriptor->name: %s\n", field_descriptor->name); + printf("field_descriptor->label: %d\n", field_descriptor->label); + + if (field_descriptor->label == PROTOBUF_C_LABEL_REQUIRED) { + result = json2protobuf_process_field(field_descriptor, json_object_value, protobuf_value, error_string, error_size); + if (result) { + SAFE_FREE_BITMAP_AND_MESSAGE; + + return result; + } + } else if (field_descriptor->label == PROTOBUF_C_LABEL_OPTIONAL) { + // Proto2 optional fields have presence tracking + if (field_descriptor->type == PROTOBUF_C_TYPE_MESSAGE || field_descriptor->type == PROTOBUF_C_TYPE_STRING) { + // Messages and strings are considered set when their pointer is non-null + } else { + // Scalar optional fields use the quantifier for presence + *(protobuf_c_boolean *)protobuf_value_quantifier = 1; + } + + result = json2protobuf_process_field(field_descriptor, json_object_value, protobuf_value, error_string, error_size); + if (result) { + SAFE_FREE_BITMAP_AND_MESSAGE; + return result; + } + } else if (field_descriptor->label == PROTOBUF_C_LABEL_NONE) { + // Proto3 fields don't track presence except for messages + result = json2protobuf_process_field(field_descriptor, json_object_value, protobuf_value, error_string, error_size); + if (result) { + SAFE_FREE_BITMAP_AND_MESSAGE; + return result; + } + } else if (field_descriptor->label == PROTOBUF_C_LABEL_REPEATED) { + if (!json_is_array(json_object_value)) { + SAFE_FREE_BITMAP_AND_MESSAGE; + + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_IS_NOT_ARRAY, + "JSON is not an array required for repeatable GPB field"); + } + + size_t *protobuf_values_count = (size_t *)protobuf_value_quantifier; + + *protobuf_values_count = json_array_size(json_object_value); + + if (*protobuf_values_count) { + size_t value_size = protobuf2json_value_size_by_type(field_descriptor->type); + if (!value_size) { + SAFE_FREE_BITMAP_AND_MESSAGE; + + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_UNSUPPORTED_FIELD_TYPE, + "Cannot calculate value size for %d using protobuf2json_value_size_by_type()", + field_descriptor->type); + } + + void *protobuf_value_repeated = calloc(*protobuf_values_count, value_size); + if (!protobuf_value_repeated) { + SAFE_FREE_BITMAP_AND_MESSAGE; + + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_ALLOCATE_MEMORY, + "Cannot allocate %zu bytes using calloc(3)", + (size_t)*protobuf_values_count * value_size); + } + + size_t json_index; + json_t *json_array_value; + json_array_foreach(json_object_value, json_index, json_array_value) { + char *protobuf_value_repeated_value = (char *)protobuf_value_repeated + json_index * value_size; + + result = json2protobuf_process_field(field_descriptor, json_array_value, (void *)protobuf_value_repeated_value, error_string, error_size); + if (result) { + /* Free already processed repeated field items */ + { + if (field_descriptor->type == PROTOBUF_C_TYPE_STRING) { + size_t t; + for (t = 0; t <= json_index; t++) { + free(((char **)protobuf_value_repeated)[t]); + } + } else if (field_descriptor->type == PROTOBUF_C_TYPE_BYTES) { + size_t t; + for (t = 0; t <= json_index; t++) { + free(((ProtobufCBinaryData *)protobuf_value_repeated)[t].data); + } + } else if (field_descriptor->type == PROTOBUF_C_TYPE_MESSAGE) { + size_t t; + for (t = 0; t <= json_index; t++) { + if (((ProtobufCMessage **)protobuf_value_repeated)[t]) { + protobuf_c_message_free_unpacked( + ((ProtobufCMessage **)protobuf_value_repeated)[t], + NULL); + } + } + } + + free(protobuf_value_repeated); + *protobuf_values_count = 0; + } + + SAFE_FREE_BITMAP_AND_MESSAGE; + + return result; + } + } + + memcpy(protobuf_value, &protobuf_value_repeated, sizeof(protobuf_value_repeated)); + } + } else { + SAFE_FREE_BITMAP_AND_MESSAGE; + + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_UNSUPPORTED_FIELD_TYPE, + "Unsupported field label %d for field '%s' in message '%s'", + field_descriptor->label, field_descriptor->name, protobuf_message_descriptor->name); + } + } + + unsigned int i = 0; + for (i = 0; i < protobuf_message_descriptor->n_fields; i++) { + const ProtobufCFieldDescriptor *field_descriptor = protobuf_message_descriptor->fields + i; + + if ((field_descriptor->label == PROTOBUF_C_LABEL_REQUIRED) && !field_descriptor->default_value && !bitmap_get(presented_fields, i)) { + SAFE_FREE_BITMAP_AND_MESSAGE; + + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_REQUIRED_IS_MISSING, + "Required field '%s' is missing in message '%s'", + field_descriptor->name, protobuf_message_descriptor->name); + } + } + + bitmap_free(presented_fields); + + return 0; +} + +/* === JSON -> Protobuf === Public === */ + +int json2protobuf_object( + json_t *json_object, + const ProtobufCMessageDescriptor *protobuf_message_descriptor, + ProtobufCMessage **protobuf_message, + char *error_string, + size_t error_size) { + int result = json2protobuf_process_message(json_object, protobuf_message_descriptor, protobuf_message, error_string, error_size); + if (result) { + return result; + } + + return 0; +} + +int json2protobuf_string( + char *json_string, + size_t json_flags, + const ProtobufCMessageDescriptor *protobuf_message_descriptor, + ProtobufCMessage **protobuf_message, + char *error_string, + size_t error_size) { + json_t *json_object = NULL; + json_error_t error; + + json_object = json_loads(json_string, json_flags, &error); + if (!json_object) { + json_decref(json_object); + + SET_ERROR_STRING_AND_RETURN( + PROTOBUF2JSON_ERR_CANNOT_PARSE_STRING, + "JSON parsing error at line %d column %d (position %d): %s", + error.line, error.column, error.position, error.text); + } + + int result = json2protobuf_object(json_object, protobuf_message_descriptor, protobuf_message, error_string, error_size); + if (result) { + json_decref(json_object); + return result; + } + + json_decref(json_object); + return 0; +} + +/* === END === */ \ No newline at end of file diff --git a/packages/pg-parser/src/index.ts b/packages/pg-parser/src/index.ts index 1034f8e..5548fd0 100644 --- a/packages/pg-parser/src/index.ts +++ b/packages/pg-parser/src/index.ts @@ -1,3 +1,3 @@ export * from './errors.js'; export * from './pg-parser.js'; -export { unwrapResult } from './util.js'; +export { unwrapParseResult as unwrapResult } from './util.js'; diff --git a/packages/pg-parser/src/pg-parser.test.ts b/packages/pg-parser/src/pg-parser.test.ts index 4e61ca1..e712470 100644 --- a/packages/pg-parser/src/pg-parser.test.ts +++ b/packages/pg-parser/src/pg-parser.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import { PgParser } from './pg-parser.js'; -import { unwrapResult } from './util.js'; +import { unwrapDeparseResult, unwrapParseResult } from './util.js'; describe('versions', () => { // it('parses sql in v15', async () => { @@ -17,13 +17,13 @@ describe('versions', () => { it('parses sql in v17', async () => { const pgParser = new PgParser({ version: 17 }); - const result = await unwrapResult(pgParser.parse('SELECT 1+1 as sum')); + const result = await unwrapParseResult(pgParser.parse('SELECT 1+1 as sum')); expect(result.version).toBe(170004); }); it('parses sql in v17 by default', async () => { const pgParser = new PgParser(); - const result = await unwrapResult(pgParser.parse('SELECT 1+1 as sum')); + const result = await unwrapParseResult(pgParser.parse('SELECT 1+1 as sum')); expect(result.version).toBe(170004); }); @@ -36,7 +36,7 @@ describe('versions', () => { describe('parser', () => { it('parses sql into ast', async () => { const pgParser = new PgParser(); - const result = await unwrapResult(pgParser.parse('SELECT 1+1 as sum')); + const result = await unwrapParseResult(pgParser.parse('SELECT 1+1 as sum')); expect(result).toMatchObject({ stmts: [ @@ -85,7 +85,7 @@ describe('parser', () => { it('parse result matches types', async () => { const pgParser = new PgParser(); - const result = await unwrapResult(pgParser.parse('SELECT 1+1 as sum')); + const result = await unwrapParseResult(pgParser.parse('SELECT 1+1 as sum')); expect(result.version).toBe(170004); @@ -193,7 +193,20 @@ describe('parser', () => { it('throws error for invalid sql', async () => { const pgParser = new PgParser(); - const resultPromise = unwrapResult(pgParser.parse('my invalid sql')); + const resultPromise = unwrapParseResult(pgParser.parse('my invalid sql')); await expect(resultPromise).rejects.toThrow('syntax error at or near "my"'); }); }); + +describe('deparser', () => { + it('deparses ast into sql', async () => { + const pgParser = new PgParser(); + const parseResult = await unwrapParseResult( + pgParser.parse('SELECT 1 + 1 AS sum') + ); + + const sql = await unwrapDeparseResult(pgParser.deparse(parseResult)); + + expect(sql).toBe('SELECT 1 + 1 AS sum'); + }); +}); diff --git a/packages/pg-parser/src/pg-parser.ts b/packages/pg-parser/src/pg-parser.ts index d969c29..b93489b 100644 --- a/packages/pg-parser/src/pg-parser.ts +++ b/packages/pg-parser/src/pg-parser.ts @@ -1,6 +1,15 @@ import { SUPPORTED_VERSIONS } from './constants.js'; import { PgParseError } from './errors.js'; -import type { MainModule, PgParserModule, SupportedVersion } from './types.js'; +import type { + MainModule, + ParseResult, + PgDeparseResult, + PgParseResult, + PgParserModule, + SupportedVersion, +} from './types.js'; + +type Pointer = number; export type PgParserOptions = { version?: T; @@ -30,10 +39,10 @@ export class PgParser { return await createModule(); } - async parse(sql: string) { + async parse(sql: string): Promise> { const module = await this.#module; - const parseResultPtr = module.ccall( + const parseResultPtr: Pointer = module.ccall( 'parse_sql', 'number', ['string'], @@ -42,32 +51,42 @@ export class PgParser { // Parse struct PgQueryProtobufParseResult from the pointer const parseTreePtr = parseResultPtr; - const stderrBufferPtr = module.getValue(parseResultPtr + 8, 'i32'); - const errorPtr = module.getValue(parseResultPtr + 12, 'i32'); + const stderrBufferPtr: Pointer = module.getValue(parseResultPtr + 8, 'i32'); + const errorPtr: Pointer = module.getValue(parseResultPtr + 12, 'i32'); const error = errorPtr ? await this.#parsePgQueryError(errorPtr) : undefined; if (error) { + module.ccall( + 'free_parse_result', + undefined, + ['number'], + [parseResultPtr] + ); return { tree: undefined, error, }; } + if (!parseTreePtr) { + throw new Error('parse tree is undefined'); + } + const stderrBuffer = stderrBufferPtr ? module.UTF8ToString(stderrBufferPtr) : undefined; // Convert protobuf to JSON - const protobufToJsonResultPtr = module.ccall( + const protobufToJsonResultPtr: Pointer = module.ccall( 'protobuf_to_json', 'number', ['number'], [parseTreePtr] ); - const parseResult = await this.#parseProtobufToJsonResult( + const parseResult = await this.#parseProtobufToJsonResult( protobufToJsonResultPtr ); @@ -80,10 +99,85 @@ export class PgParser { }; } + async deparse(parseTree: ParseResult): Promise { + const module = await this.#module; + + // Convert JSON to protobuf + const jsonToProtobufResultPtr: Pointer = module.ccall( + 'json_to_protobuf', + 'number', + ['string'], + [JSON.stringify(parseTree)] + ); + + const protobufPtr = await this.#parseJsonToProtobufResult( + jsonToProtobufResultPtr + ); + + const deparseResultPtr: Pointer = module.ccall( + 'deparse_sql', + 'number', + ['number'], + [protobufPtr] + ); + + // Free the protobuf result after we're done with it + module.ccall( + 'free_json_to_protobuf_result', + undefined, + ['number'], + [jsonToProtobufResultPtr] + ); + + // Parse struct PgQueryDeparseResult from the pointer + const queryPtr = module.getValue(deparseResultPtr, 'i32'); + const errorPtr = module.getValue(deparseResultPtr + 4, 'i32'); + const error = errorPtr + ? await this.#parsePgQueryError(errorPtr) + : undefined; + + if (error) { + module.ccall( + 'free_deparse_result', + undefined, + ['number'], + [deparseResultPtr] + ); + return { + sql: undefined, + error, + }; + } + + const sql = queryPtr ? module.UTF8ToString(queryPtr) : undefined; + + if (!sql) { + module.ccall( + 'free_deparse_result', + undefined, + ['number'], + [deparseResultPtr] + ); + throw new Error('query is undefined'); + } + + module.ccall( + 'free_deparse_result', + undefined, + ['number'], + [deparseResultPtr] + ); + + return { + sql, + error: undefined, + }; + } + /** * Parses a ProtobufToJsonResult struct from a pointer. */ - async #parseProtobufToJsonResult(resultPtr: number): Promise { + async #parseProtobufToJsonResult(resultPtr: Pointer): Promise { const module = await this.#module; const jsonStringPtr = module.getValue(resultPtr, 'i32'); @@ -113,6 +207,22 @@ export class PgParser { return JSON.parse(jsonString); } + async #parseJsonToProtobufResult(resultPtr: Pointer): Promise { + const module = await this.#module; + + const pgQueryProtobufPtr = resultPtr; + const errorPtr: number = module.getValue(resultPtr + 8, 'i32'); + + const error = errorPtr ? module.UTF8ToString(errorPtr) : undefined; + + if (error) { + // This is unexpected, so throw instead of returning an error + throw new Error(error); + } + + return pgQueryProtobufPtr; + } + /** * Parses a PgQueryError struct from a pointer. */ diff --git a/packages/pg-parser/src/types.ts b/packages/pg-parser/src/types.ts index 37de4c2..e37b828 100644 --- a/packages/pg-parser/src/types.ts +++ b/packages/pg-parser/src/types.ts @@ -43,3 +43,15 @@ export type PgParseResultError = { export type PgParseResult = | PgParseResultSuccess | PgParseResultError; + +export type PgDeparseResultSuccess = { + sql: string; + error: undefined; +}; + +export type PgDeparseResultError = { + sql: undefined; + error: PgParseError; +}; + +export type PgDeparseResult = PgDeparseResultSuccess | PgDeparseResultError; diff --git a/packages/pg-parser/src/util.ts b/packages/pg-parser/src/util.ts index a3f4f0b..34dd5cb 100644 --- a/packages/pg-parser/src/util.ts +++ b/packages/pg-parser/src/util.ts @@ -1,6 +1,10 @@ -import type { PgParseResult, SupportedVersion } from './types.js'; +import type { + PgDeparseResult, + PgParseResult, + SupportedVersion, +} from './types.js'; -export async function unwrapResult( +export async function unwrapParseResult( result: PgParseResult | Promise> ) { let resolved = await result; @@ -9,3 +13,13 @@ export async function unwrapResult( } return resolved.tree; } + +export async function unwrapDeparseResult( + result: PgDeparseResult | Promise +) { + let resolved = await result; + if (resolved.error) { + throw resolved.error; + } + return resolved.sql; +} diff --git a/packages/pg-parser/tools/emsdk/Dockerfile b/packages/pg-parser/tools/emsdk/Dockerfile index 3fde14b..854952d 100644 --- a/packages/pg-parser/tools/emsdk/Dockerfile +++ b/packages/pg-parser/tools/emsdk/Dockerfile @@ -3,6 +3,17 @@ FROM emscripten/emsdk RUN apt-get update && apt-get install -y \ autoconf \ automake \ - libtool + libtool \ + pkg-config \ + protobuf-compiler \ + libprotoc-dev + +# Install protobuf-c +RUN git clone --branch feat/json_name https://github.com/gregnr/protobuf-c.git /protobuf-c && \ + cd /protobuf-c && \ + ./autogen.sh && \ + ./configure && \ + make -j$(nproc) && \ + make install RUN npm i -g typescript tsx