Skip to content

Commit

Permalink
chores(bq): split JS libraries generation (#541)
Browse files Browse the repository at this point in the history
  • Loading branch information
vdelacruzb authored Jan 24, 2025
1 parent 14b2b5c commit a77c453
Show file tree
Hide file tree
Showing 82 changed files with 1,400 additions and 1,230 deletions.
6 changes: 5 additions & 1 deletion clouds/bigquery/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ ROOT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))

DIST_DIR ?= $(ROOT_DIR)/dist
BUILD_DIR ?= $(ROOT_DIR)/build
MODULES_DIRS ?= $(ROOT_DIR)/modules
ESLINTRC_DIR ?= $(ROOT_DIR)/../..
COMMON_DIR = $(ROOT_DIR)/common
PACKAGE_VERSION ?= $(shell cat $(ROOT_DIR)/version)
Expand Down Expand Up @@ -49,7 +50,10 @@ build:
build-libraries:
mkdir -p $(BUILD_DIR)/libs
$(MAKE) -C libraries/javascript build
cp libraries/javascript/build/index.js $(BUILD_DIR)/libs/$(BQ_LIBRARY_DEFAULT).js
$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB) 1>/dev/null # Check errors
for f in `$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB)`; do \
cp libraries/javascript/build/$${f}.js $(BUILD_DIR)/libs/$(BQ_LIBRARY_DEFAULT)_$${f}.js; \
done
ifdef MAKE_LIB
cp libraries/javascript/build/index_$(MAKE_LIB).js $(BUILD_DIR)/libs/$(BQ_LIBRARY_DEFAULT)_$(MAKE_LIB).js
endif
Expand Down
8 changes: 4 additions & 4 deletions clouds/bigquery/common/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ BQ_DATASET_DEFAULT = carto
BQ_LIBRARY_DEFAULT ?= carto_analytics_toolbox_core

ifeq ($(production),1)
export BQ_LIBRARY_BUCKET = $(BQ_BUCKET)/$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT).js
export BQ_LIBRARY_TILER_BUCKET = $(BQ_BUCKET)/$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT)_tiler.js
export BQ_LIBRARY_BUCKET = $(BQ_BUCKET)/$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT)
export BQ_LIBRARY_TILER_BUCKET = $(BQ_LIBRARY_BUCKET)_tiler.js
else
export BQ_LIBRARY_BUCKET = $(BQ_BUCKET)/$(BQ_PREFIX)$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT).js
export BQ_LIBRARY_TILER_BUCKET = $(BQ_BUCKET)/$(BQ_PREFIX)$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT)_tiler.js
export BQ_LIBRARY_BUCKET = $(BQ_BUCKET)/$(BQ_PREFIX)$(BQ_DATASET_DEFAULT)/libs/$(BQ_LIBRARY_DEFAULT)
export BQ_LIBRARY_TILER_BUCKET = $(BQ_LIBRARY_BUCKET)_tiler.js

endif

Expand Down
20 changes: 20 additions & 0 deletions clouds/bigquery/common/build_modules.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,11 @@ const argv = require('minimist')(process.argv.slice(2));

const inputDirs = argv._[0] && argv._[0].split(',');
const outputDir = argv.output || 'build';
const libsBuildDir = argv.libs_build_dir || '../libraries/javascript/build';
const diff = argv.diff || [];
const nodeps = argv.nodeps;
const libraryBucket = argv.librarybucket;
const makelib = argv.makelib;
let modulesFilter = (argv.modules && argv.modules.split(',')) || [];
let functionsFilter = (argv.functions && argv.functions.split(',')) || [];
let all = !(diff.length || modulesFilter.length || functionsFilter.length);
Expand Down Expand Up @@ -156,6 +159,23 @@ if (argv.production) {
let content = output.map(f => f.content).join(separator);

function apply_replacements (text) {
const libraries = [... new Set(text.match(new RegExp('@@BQ_LIBRARY_.*_BUCKET@@', 'g')))];
for (let library of libraries) {
let libraryName = library.replace('@@BQ_LIBRARY_', '').replace('_BUCKET@@', '').toLowerCase();
if (makelib == libraryName) {
continue;
}
libraryName += '.js';
const libraryPath = path.join(libsBuildDir, libraryName);
if (fs.existsSync(libraryPath)) {
const libraryBucketPath = libraryBucket + '_' + libraryName;
text = text.replace(new RegExp(library, 'g'), libraryBucketPath);
}
else {
console.log(`Warning: library "${libraryName}" does not exist. Run "make build-libraries" with the same filters.`);
process.exit(1);
}
}
const replacements = process.env.REPLACEMENTS.split(' ');
for (let replacement of replacements) {
if (replacement) {
Expand Down
145 changes: 145 additions & 0 deletions clouds/bigquery/common/list_libraries.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
#!/usr/bin/env node

// List the JavaScript libraries based on the input filters to the SQL functions


// ./build_modules.js modules --output=build --diff="clouds/bigquery/modules/sql/quadbin/QUADBIN_TOZXY.sql"
// ./build_modules.js modules --output=build --functions=ST_TILEENVELOPE
// ./build_modules.js modules --output=build --modules=quadbin
// ./build_modules.js modules --output=build --production --dropfirst

const fs = require('fs');
const path = require('path');
const argv = require('minimist')(process.argv.slice(2));

const inputDirs = argv._[0] && argv._[0].split(',');
const diff = argv.diff || [];
const nodeps = argv.nodeps;
const makelib = argv.makelib;
let modulesFilter = (argv.modules && argv.modules.split(',')) || [];
let functionsFilter = (argv.functions && argv.functions.split(',')) || [];
let all = !(diff.length || modulesFilter.length || functionsFilter.length);

// Convert diff to modules/functions
if (diff.length) {
const patternsAll = [
/\.github\/workflows\/bigquery\.yml/,
/clouds\/bigquery\/common\/.+/,
/clouds\/bigquery\/libraries\/.+/,
/clouds\/bigquery\/.*Makefile/,
/clouds\/bigquery\/version/
];
const patternModulesSql = /clouds\/bigquery\/modules\/sql\/([^\s]*?)\//g;
const patternModulesTest = /clouds\/bigquery\/modules\/test\/([^\s]*?)\//g;
const diffAll = patternsAll.some(p => diff.match(p));
if (diffAll) {
all = diffAll;
} else {
const modulesSql = [...diff.matchAll(patternModulesSql)].map(m => m[1]);
const modulesTest = [...diff.matchAll(patternModulesTest)].map(m => m[1]);
const diffModulesFilter = [...new Set(modulesSql.concat(modulesTest))];
if (diffModulesFilter) {
modulesFilter = diffModulesFilter;
}
}
}

// Extract functions
const functions = [];
for (let inputDir of inputDirs) {
const sqldir = path.join(inputDir, 'sql');
const modules = fs.readdirSync(sqldir);
modules.forEach(module => {
const moduledir = path.join(sqldir, module);
if (fs.statSync(moduledir).isDirectory()) {
const files = fs.readdirSync(moduledir);
files.forEach(file => {
if (file.endsWith('.sql')) {
const name = path.parse(file).name;
const content = fs.readFileSync(path.join(moduledir, file)).toString().replace(/--.*\n/g, '');
functions.push({
name,
module,
content,
dependencies: []
});
}
});
}
});
}

// Check filters
modulesFilter.forEach(m => {
if (!functions.map(fn => fn.module).includes(m)) {
process.stderr.write(`ERROR: Module not found ${m}\n`);
process.exit(1);
}
});
functionsFilter.forEach(f => {
if (!functions.map(fn => fn.name).includes(f)) {
process.stderr.write(`ERROR: Function not found ${f}`);
process.exit(1);
}
});

// Extract function dependencies
if (!nodeps) {
functions.forEach(mainFunction => {
functions.forEach(depFunction => {
if (mainFunction.name != depFunction.name) {
const depFunctionMatches = [];
depFunctionMatches.push(...depFunction.content.replace(/(\r\n|\n|\r)/gm,' ').matchAll(new RegExp('(?<=(?<!TEMP )FUNCTION)(.*?)(?=AS |RETURNS)','g')));
depFunctionMatches.push(...depFunction.content.replace(/(\r\n|\n|\r)/gm,' ').matchAll(new RegExp('(?<=PROCEDURE)(.*?)(?=BEGIN)','g')));
const depFunctionNames = [];
depFunctionMatches.forEach((depFunctionMatch) => {
let qualifiedDepFunctName = depFunctionMatch[0].replace(/[ \p{Diacritic}]/gu, '').split('(')[0];
qualifiedDepFunctName = qualifiedDepFunctName.split('.');
depFunctionNames.push(qualifiedDepFunctName[qualifiedDepFunctName.length - 1]);
})
if (depFunctionNames.some((depFunctionName) => mainFunction.content.includes(`DATASET@@.${depFunctionName}\`(`))) {
mainFunction.dependencies.push(depFunction.name);
}
}
});
});
}

// Check circular dependencies
functions.forEach(mainFunction => {
functions.forEach(depFunction => {
if (mainFunction.dependencies.includes(depFunction.name) &&
depFunction.dependencies.includes(mainFunction.name)) {
process.stderr.write(`ERROR: Circular dependency between ${mainFunction.name} and ${depFunction.name}`);
process.exit(1);
}
});
});


// Filter functions
const output = [];
function add (f, include) {
include = include || all || functionsFilter.includes(f.name) || modulesFilter.includes(f.module);
for (const dependency of f.dependencies) {
add(functions.find(f => f.name === dependency), include);
}
if (!output.map(f => f.name).includes(f.name) && include) {
output.push({
name: f.name,
content: f.content
});
}
}
functions.forEach(f => add(f));

const content = output.map(f => f.content).join('\n');
let libraries = [... new Set(content.match(new RegExp('@@BQ_LIBRARY_.*_BUCKET@@', 'g')))]
.map(l => l.replace('@@BQ_LIBRARY_', '').replace('_BUCKET@@', '').toLowerCase());

// Exclude libraries pointed by makelib as they are deployed separately
if (makelib) {
libraries = libraries.filter(l => l !== makelib);
}

process.stdout.write(libraries.join(' '));
5 changes: 5 additions & 0 deletions clouds/bigquery/common/rollup.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,11 @@ for (let dir of dirs) {
}
}

if (!input && filename) {
console.log(`Error: library "${filename}" does not exist. Add it or revisit the replacement "@@BQ_LIBRARY_${path.parse(filename).name.toUpperCase()}@@" in one of your sql files.`);
process.exit(1);
}

// Format library name to camel case
const name = process.env.NAME.replace(/(_\w)/g, k => k[1].toUpperCase());

Expand Down
32 changes: 25 additions & 7 deletions clouds/bigquery/libraries/javascript/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,27 @@ ifdef MAKE_LIB
endif

build-libs: build-install $(NODE_MODULES_DEV)
NAME=lib \
PATH="$(NODE_MODULES_DEV)/.bin/:$(PATH)" \
DIRS=$(LIBS_DIR) \
FILENAME=index.js \
OUTPUT=$(BUILD_DIR)/index.js \
rollup --config $(COMMON_DIR)/rollup.config.js $(BUILD_PARAMS); \
ifdef UNIT_TEST
$(COMMON_DIR)/list_libraries.js $(MODULES_DIR) --all --makelib=$(MAKE_LIB) 1>/dev/null # Check errors
for f in `$(COMMON_DIR)/list_libraries.js $(MODULES_DIR) --all --makelib=$(MAKE_LIB)`; do \
NAME=$${f}Lib \
PATH="$(NODE_MODULES_DEV)/.bin/:$(PATH)" \
DIRS=$(LIBS_DIR) \
FILENAME=$${f}.js \
OUTPUT=$(BUILD_DIR)/$${f}.js \
rollup --config $(COMMON_DIR)/rollup.config.js $(BUILD_PARAMS); \
done
else
$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB) 1>/dev/null # Check errors
for f in `$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB)`; do \
NAME=$${f}Lib \
PATH="$(NODE_MODULES_DEV)/.bin/:$(PATH)" \
DIRS=$(LIBS_DIRS) \
FILENAME=$${f}.js \
OUTPUT=$(BUILD_DIR)/$${f}.js \
rollup --config $(COMMON_DIR)/rollup.config.js $(BUILD_PARAMS); \
done
endif

build-install:
for d in $(shell echo $(LIBS_DIRS) | tr "," "\n"); do \
Expand All @@ -59,7 +74,10 @@ build-install:

deploy: check build
echo "Deploying libraries..."
$(GSUTIL) cp -r $(BUILD_DIR)/index.js $(BQ_LIBRARY_BUCKET)
$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB) 1>/dev/null # Check errors
for f in `$(COMMON_DIR)/list_libraries.js $(MODULES_DIRS) --diff="$(diff)" --modules=$(modules) --functions=$(functions) --nodeps=$(nodeps) --makelib=$(MAKE_LIB)`; do \
$(GSUTIL) cp -r $(BUILD_DIR)/$${f}.js $(BQ_LIBRARY_BUCKET)_$${f}.js; \
done
# Provisional for WASM version of tiler
ifdef MAKE_LIB
echo "Deploying tiler library..."
Expand Down
25 changes: 0 additions & 25 deletions clouds/bigquery/libraries/javascript/libs/index.js

This file was deleted.

25 changes: 25 additions & 0 deletions clouds/bigquery/libraries/javascript/libs/quadkey.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import {
bbox,
toParent,
toChildren,
quadkeyFromQuadint,
quadintFromQuadkey,
quadintFromLocation,
quadintToGeoJSON,
quadintFromZXY,
geojsonToQuadints,
ZXYFromQuadint
} from '../src/quadkey';

export default {
bbox,
toParent,
toChildren,
quadkeyFromQuadint,
quadintFromQuadkey,
quadintFromLocation,
quadintToGeoJSON,
quadintFromZXY,
geojsonToQuadints,
ZXYFromQuadint
};
8 changes: 8 additions & 0 deletions clouds/bigquery/libraries/javascript/libs/random.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import {
generateRandomPointsInPolygon

} from '../src/random';

export default {
generateRandomPointsInPolygon
};
9 changes: 9 additions & 0 deletions clouds/bigquery/libraries/javascript/libs/s2.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import { S2 } from '../src/s2';

export default {
keyToId: S2.keyToId,
idToKey: S2.idToKey,
latLngToKey: S2.latLngToKey,
FromHilbertQuadKey: S2.S2Cell.FromHilbertQuadKey,
idToLatLng: S2.idToLatLng
};
Loading

0 comments on commit a77c453

Please sign in to comment.