Skip to content

fix: handle conversion of native geometry columns #116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
saidy-moregeo wants to merge 5 commits into
base: main
Choose a base branch
from
Open

fix: handle conversion of native geometry columns #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
be4c31d
fix: handle conversion of native geometry columns
saidy-moregeo Jun 5, 2026
1bdf551
chore: hide conversion feature for local files
saidy-moregeo Jun 5, 2026
1acc8e1
chore: lint code
saidy-moregeo Jun 5, 2026
85f1ffa
chore: refactor
saidy-moregeo Jun 5, 2026
0590f1c
chore: explicitly add native geoarrow on the list of encodings
saidy-moregeo Jun 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 13 additions & 8 deletions src/App.vue
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -466,6 +466,11 @@ export default {
if (!this.needsReprojection) return null;
return JSON.stringify(this.primaryGeoCrs);
},
/** Geometry encoding for the primary geo column ('wkb' or 'wkt') */
geomEncoding() {
if (!this.geoMetadata?.columns || !this.primaryGeoColumn) return null;
return this.geoMetadata.columns[this.primaryGeoColumn]?.encoding ?? null;
},
/** Whether the primary geo column has covering/bbox metadata */
hasBboxCovering() {
if (!this.geoMetadata?.columns || !this.primaryGeoColumn) return false;
Expand Down Expand Up @@ -627,12 +632,13 @@ export default {
this.loadDialogOpen = true;
}
},
this.source && {
title: 'Convert',
action: () => {
this.convertDialogOpen = true;
this.source &&
!this.localFileName && {
title: 'Convert',
action: () => {
this.convertDialogOpen = true;
}
}
}
].filter(Boolean)
},
(this.fileInfo || this.schema || this.kvMetadata || this.geoMetadata) && {
Expand Down Expand Up @@ -1043,8 +1049,6 @@ export default {
// Avoids SELECT * which fetches bbox structs, binary blobs, etc.
const tableColNames = this.visibleColumns.map((c) => c.name);
const geoCol = this.loadGeometry ? this.primaryGeoColumn : null;

const colMeta = this.geoMetadata?.columns?.[geoCol];
const selectColumns = geoCol ? [...tableColNames, geoCol] : tableColNames;

if (tableColNames.length === 0 && !geoCol) {
Expand All @@ -1059,7 +1063,7 @@ export default {
const result = await queryData(this.source, {
geoColumn: geoCol,
filters: this.filters,
encoding: colMeta?.encoding || null,
encoding: this.geomEncoding,
bbox: this.spatialFilterActive ? this.viewportBounds : null,
sourceCrs: this.sourceCrsString,
columns: selectColumns,
Expand Down Expand Up @@ -1243,6 +1247,7 @@ export default {
format,
outputName,
schema: this.schema,
encoding: this.geomEncoding,
geoColumns: this.geoColumns,
primaryGeoColumn: this.primaryGeoColumn,
sourceCrs: this.sourceCrsString,
Expand Down
4 changes: 4 additions & 0 deletions src/converter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ export const FORMATS = [
* @param {Array} opts.schema - Schema columns ({name, type, ...}).
* @param {Array<string>} opts.geoColumns - Names of geometry columns.
* @param {string|null} opts.primaryGeoColumn - Primary geometry column name.
* @param {string|null} opts.encoding - Geometry encoding ('wkb', 'wkt', or 'native geoArrow').

@m-mohr m-mohr Jun 8, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So was pass in "native geoArrow"? I thought it would be "point", "polygon", etc...?

@saidy-moregeo saidy-moregeo Jun 8, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, isn't the list (points, linestrings, multipoints, etc )considered native geoArrow.

@saidy-moregeo saidy-moregeo Jun 8, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can update it. I just thought native geoArrow reflects the list that's not wkt or wkb.

@m-mohr m-mohr Jun 8, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should reflect the API as is.


* @param {string|null} opts.sourceCrs - Source CRS as PROJJSON string (null = WGS84).
* @param {Function} opts.onStatus - status(message) callback.
* @returns {{ promise: Promise<{filename:string}>, cancel: Function }}
Expand All @@ -42,6 +44,7 @@ export function startConversion({
schema,
geoColumns = [],
primaryGeoColumn = null,
encoding = null,
sourceCrs = null,
onStatus = () => {}
}) {
Expand Down Expand Up @@ -97,6 +100,7 @@ export function startConversion({
{
type: 'convert',
source,
encoding,
sourceBuffer,
sourceName: sourceBuffer ? `convert_src_${Date.now()}.parquet` : null,
format,
Expand Down
40 changes: 39 additions & 1 deletion src/converter.worker.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,43 @@ async function detectAlreadyGeometry(escaped, geoColumn) {
return false;
}

function buildGeoExpr(geomCol, encoding) {
const col = quoteIdent(geomCol);
const ptToWkt = `pt -> CAST(pt.x AS VARCHAR) || ' ' || CAST(pt.y AS VARCHAR)`;
const ringToWkt = (inner) =>
`ring -> '(' || array_to_string(list_transform(ring, ${inner}), ', ') || ')'`;
const wrapGeom = (wktExpr) =>
`CASE WHEN ${col} IS NOT NULL AND len(${col}) > 0 THEN ST_GeomFromText(${wktExpr}) ELSE NULL END`;
const arr = (expr, lambda) => `array_to_string(list_transform(${expr}, ${lambda}), ', ')`;

switch (encoding?.toLowerCase()) {
case 'wkb':
return `${col})`;
case 'wkt':
return `ST_GeomFromText(${col})`;
case 'point':
return `CASE WHEN ${col} IS NOT NULL AND ${col}.x IS NOT NULL THEN ST_Point(${col}.x, ${col}.y) ELSE NULL END`;
case 'linestring':
return wrapGeom(`'LINESTRING(' || ${arr(col, ptToWkt)} || ')'`);
case 'polygon':
return wrapGeom(`'POLYGON(' || ${arr(col, ringToWkt(ptToWkt))} || ')'`);
case 'multipoint':
return wrapGeom(
`'MULTIPOINT(' || ${arr(col, `pt -> '(' || CAST(pt.x AS VARCHAR) || ' ' || CAST(pt.y AS VARCHAR) || ')'`)} || ')'`
);
case 'multilinestring':
return wrapGeom(
`'MULTILINESTRING(' || ${arr(col, `line -> '(' || ${arr('line', ptToWkt)} || ')'`)} || ')'`
);
case 'multipolygon':
return wrapGeom(
`'MULTIPOLYGON(' || ${arr(col, `poly -> '(' || ${arr('poly', ringToWkt(ptToWkt))} || ')'`)} || ')'`
);
default:
return `ST_GeomFromWKB(${col})`;
}
}

self.onmessage = async (ev) => {
const msg = ev.data;
if (!msg || msg.type !== 'convert') return;
Expand All @@ -107,6 +144,7 @@ self.onmessage = async (ev) => {

async function convert({
source,
encoding,
sourceBuffer,
sourceName,
format,
Expand Down Expand Up @@ -142,7 +180,7 @@ async function convert({
if (primaryGeoColumn) {
const base = alreadyGeometry
? quoteIdent(primaryGeoColumn)
: `ST_GeomFromWKB(${quoteIdent(primaryGeoColumn)})`;
: buildGeoExpr(primaryGeoColumn, encoding);
if (sourceCrs) {
const lit = sourceCrs.replace(/'/g, "''");
geomExpr = `ST_Transform(${base}, '${lit}', 'EPSG:4326', true)`;
Expand Down
27 changes: 14 additions & 13 deletions src/db.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,11 @@ function geomExpr(geoColumn, encoding) {
const col = `"${geoColumn}"`;
// Lambda fragment: converts a point struct {x, y} to a WKT coordinate string "x y".
const ptToWkt = `pt -> CAST(pt.x AS VARCHAR) || ' ' || CAST(pt.y AS VARCHAR)`;
const ringToWkt = (inner) =>
`ring -> '(' || array_to_string(list_transform(ring, ${inner}), ', ') || ')'`;
const wrapGeom = (wktExpr) =>
`CASE WHEN ${col} IS NOT NULL AND len(${col}) > 0 THEN ST_GeomFromText(${wktExpr}) ELSE NULL END`;
const arr = (expr, lambda) => `array_to_string(list_transform(${expr}, ${lambda}), ', ')`;
Comment on lines +188 to +192

@m-mohr m-mohr Jun 8, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What does this do?

@saidy-moregeo saidy-moregeo Jun 8, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's an abstraction helper functions to avoid having to repeated the same code multiple times on the different cases of the native geoArrow.

ringToWkt: Helper to transform list of points or lines into lambda strings that can be easily converted to duckDB's Geometry type.
wrapGeom: Checks for nulls before converting the constructed wkt to the actual duckDb's geometry type that can be transform.
arr : Helper function to map each element of a list to a lambda that's join in a comma separated list.

@m-mohr m-mohr Jun 8, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay. Any reason these are redefined on each function call instead of defining them once?

@saidy-moregeo saidy-moregeo Jun 8, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you are referring to having them defined on both the db.js and converter.worker.js, frankly was planning to have a single buildGeomExpr function that can be invoked on either files to avoid multiple definition, but didn't follow through as was experience some errors after refactor and I decided to not waste more time on such.

@m-mohr m-mohr Jun 8, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, I meant why you use inline functions.


switch (encoding?.toLowerCase()) {
case 'wkb':
Expand All @@ -204,32 +209,28 @@ function geomExpr(geoColumn, encoding) {

case 'linestring':
// LIST(STRUCT(x,y)) → LINESTRING(x1 y1, x2 y2, ...)
return `CASE WHEN ${col} IS NOT NULL AND len(${col}) > 0 THEN ST_GeomFromText('LINESTRING(' || array_to_string(list_transform(${col}, ${ptToWkt}), ', ') || ')') ELSE NULL END`;
return wrapGeom(`'LINESTRING(' || ${arr(col, ptToWkt)} || ')'`);

case 'polygon':
// LIST(LIST(STRUCT(x,y))) → POLYGON((x1 y1, ...), (x2 y2, ...))
return (
`CASE WHEN ${col} IS NOT NULL AND len(${col}) > 0 THEN ST_GeomFromText('POLYGON(' || array_to_string(list_transform(${col}, ` +
`ring -> '(' || array_to_string(list_transform(ring, ${ptToWkt}), ', ') || ')'), ', ') || ')') ELSE NULL END`
);
return wrapGeom(`'POLYGON(' || ${arr(col, ringToWkt(ptToWkt))} || ')'`);

case 'multipoint':
// LIST(STRUCT(x,y)) → MULTIPOINT((x1 y1), (x2 y2), ...)
return `CASE WHEN ${col} IS NOT NULL AND len(${col}) > 0 THEN ST_GeomFromText('MULTIPOINT(' || array_to_string(list_transform(${col}, pt -> '(' || CAST(pt.x AS VARCHAR) || ' ' || CAST(pt.y AS VARCHAR) || ')'), ', ') || ')') ELSE NULL END`;
return wrapGeom(
`'MULTIPOINT(' || ${arr(col, `pt -> '(' || CAST(pt.x AS VARCHAR) || ' ' || CAST(pt.y AS VARCHAR) || ')'`)} || ')'`
);

case 'multilinestring':
// LIST(LIST(STRUCT(x,y))) → MULTILINESTRING((x1 y1, x2 y2), (...))
return (
`CASE WHEN ${col} IS NOT NULL AND len(${col}) > 0 THEN ST_GeomFromText('MULTILINESTRING(' || array_to_string(list_transform(${col}, ` +
`line -> '(' || array_to_string(list_transform(line, ${ptToWkt}), ', ') || ')'), ', ') || ')') ELSE NULL END`
return wrapGeom(
`'MULTILINESTRING(' || ${arr(col, `line -> '(' || ${arr('line', ptToWkt)} || ')'`)} || ')'`
);

case 'multipolygon':
// LIST(LIST(LIST(STRUCT(x,y)))) → MULTIPOLYGON(((x y, ...), (...)), ((...)))
return (
`CASE WHEN ${col} IS NOT NULL AND len(${col}) > 0 THEN ST_GeomFromText('MULTIPOLYGON(' || array_to_string(list_transform(${col}, ` +
`poly -> '(' || array_to_string(list_transform(poly, ` +
`ring -> '(' || array_to_string(list_transform(ring, ${ptToWkt}), ', ') || ')'), ', ') || ')'), ', ') || ')') ELSE NULL END`
return wrapGeom(
`'MULTIPOLYGON(' || ${arr(col, `poly -> '(' || ${arr('poly', ringToWkt(ptToWkt))} || ')'`)} || ')'`
);

default:
Expand Down