release: 2024-01-17 (#471)

Co-authored-by: Javier Goizueta <[email protected]>
Co-authored-by: Alberto Hernández <[email protected]>
Co-authored-by: Jesús Arroyo Torrens <[email protected]>
Co-authored-by: Pedro-Juan Ferrer <[email protected]>

.github/workflows/bigquery-ded.yml

@@ -27,15 +27,11 @@ jobs:
       - name: Set BQ_PREFIX for releases
         if: startsWith(github.event.pull_request.head.ref, 'release/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#release/}
-          echo "BQ_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "BQ_PREFIX=dedicated_release_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set BQ_PREFIX for hotfixes
         if: startsWith(github.event.pull_request.head.ref, 'hotfix/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#hotfix/}
-          echo "BQ_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "BQ_PREFIX=dedicated_hotfix_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set BQ_PREFIX for regular deploys
         if: |
           !(startsWith(github.event.pull_request.head.ref, 'hotfix/')) &&

.github/workflows/postgres-ded.yml

@@ -29,15 +29,11 @@ jobs:
       - name: Set PG_PREFIX for releases
         if: startsWith(github.event.pull_request.head.ref, 'release/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#release/}
-          echo "PG_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "PG_PREFIX=dedicated_release_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set PG_PREFIX for hotfixes
         if: startsWith(github.event.pull_request.head.ref, 'hotfix/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#hotfix/}
-          echo "PG_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "PG_PREFIX=dedicated_hotfix_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set PG_PREFIX for regular deploys
         if: |
           !(startsWith(github.event.pull_request.head.ref, 'hotfix/')) &&

.github/workflows/redshift-ded.yml

@@ -18,6 +18,7 @@ jobs:
       (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'dedicated_redshift')) ||
       (github.event.action == 'synchronize' && contains(github.event.pull_request.labels.*.name, 'dedicated_redshift'))
     runs-on: ubuntu-20.04
+    container: python:2.7.18-buster
     timeout-minutes: 20
     env:
       RS_HOST: ${{ secrets.RS_HOST_CD }}
@@ -30,23 +31,8 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@v2
-      - name: Set RS_PREFIX for releases
-        if: startsWith(github.event.pull_request.head.ref, 'release/')
-        run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#release/}
-          echo "RS_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
-      - name: Set RS_PREFIX for hotfixes
-        if: startsWith(github.event.pull_request.head.ref, 'hotfix/')
-        run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#hotfix/}
-          echo "RS_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
-      - name: Set RS_PREFIX for regular deploys
-        if: |
-          !(startsWith(github.event.pull_request.head.ref, 'hotfix/')) &&
-          !(startsWith(github.event.pull_request.head.ref, 'release/'))
-        run: echo "RS_PREFIX=dedicated_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
+      - name: Check diff
+        uses: technote-space/get-diff-action@v4
       - name: Setup node
         uses: actions/setup-node@v1
         with:
@@ -62,6 +48,8 @@ jobs:
           curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
           unzip awscliv2.zip
           ./aws/install
+      - name: Setup virtualenv
+        run: pip install virtualenv==${{ env.VIRTUALENV_VERSION }}
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:

.github/workflows/snowflake-ded.yml

@@ -5,7 +5,7 @@ on:
     types: [closed, unlabeled, labeled, synchronize]
 
 env:
-  NODE_VERSION: 14
+  NODE_VERSION: 16
 
 jobs:
 
@@ -28,15 +28,11 @@ jobs:
       - name: Set SF_PREFIX for releases
         if: startsWith(github.event.pull_request.head.ref, 'release/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#release/}
-          echo "SF_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "SF_PREFIX=dedicated_release_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set SF_PREFIX for hotfixes
         if: startsWith(github.event.pull_request.head.ref, 'hotfix/')
         run: |
-          BRANCH_NAME="${{ github.event.pull_request.head.ref }}"
-          VERSION=${BRANCH_NAME#hotfix/}
-          echo "SF_PREFIX=dedicated_release_${VERSION}_" >> $GITHUB_ENV
+          echo "SF_PREFIX=dedicated_hotfix_${{ github.event.pull_request.number }}_" >> $GITHUB_ENV
       - name: Set SF_PREFIX for regular deploys
         if: |
           !(startsWith(github.event.pull_request.head.ref, 'hotfix/')) &&

.github/workflows/snowflake.yml

@@ -15,7 +15,7 @@ on:
   workflow_call:
 
 env:
-  NODE_VERSION: 14
+  NODE_VERSION: 16
   PYTHON3_VERSION: 3.8.10
   VIRTUALENV_VERSION: 20.21.1
   GCLOUD_VERSION: 290.0.1
@@ -88,6 +88,37 @@ jobs:
           cd clouds/snowflake
           make deploy diff="$GIT_DIFF" production=1
 
+  deploy-internal-app:
+    if: github.ref_name == 'main'
+    needs: test
+    runs-on: ubuntu-20.04
+    timeout-minutes: 20
+    env:
+      APP_PACKAGE_NAME: ${{ secrets.SF_NATIVE_APP_PACKAGE_NAME_CD }}
+      APP_NAME: ${{ secrets.SF_NATIVE_APP_NAME_CD }}
+      SF_ACCOUNT: ${{ secrets.SF_ACCOUNT_NATIVE_APP }}
+      SF_DATABASE: ${{ secrets.SF_DATABASE_NATIVE_APP }}
+      SF_USER: ${{ secrets.SF_USER_NATIVE_APP }}
+      SF_PASSWORD: ${{ secrets.SF_PASSWORD_NATIVE_APP }}
+      SF_ROLE: ${{ secrets.SF_ROLE_NATIVE_APP }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+      - name: Check diff
+        uses: technote-space/get-diff-action@v4
+      - name: Setup node
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+      - name: Deploy native app package
+        run: |
+          cd clouds/snowflake
+          make deploy-native-app-package production=1
+      - name: Deploy native app locally
+        run: |
+          cd clouds/snowflake
+          make deploy-native-app production=1
+
   deploy-share:
     if: github.ref_name == 'stable'
     needs: test

CHANGELOG.md

@@ -4,6 +4,19 @@ CARTO Analytics Toolbox Core.
 
 All notable commits to this project will be documented in this file.
 
+## 2024-01-17
+
+- chore(bq): increase tests timeout to 200000 (#455)
+- feat(bq,sf,rs,pg|quadbin): add function QUADBIN_DISTANCE (#457)
+- fix(bq|h3): fix broken reference in H3_POLYFILL_TABLE (#458, #460)
+- chore(bq,sf,rs,pg): fix naming dedicated deployments for releases (#462)
+- fix(sf|quadbin): QUADBIN_FROMLONGLAT not clamping latitudes and return some quadbin functions return NULL when NULL parameters (#456)
+- fix(rs|constructors,transformations): adjust SRID and use native ST_GEOMFROMGEOJSON to return geometries instead of VARCHAR (#463)
+- chore(pg): fix typo naming dedicated deployments for releases (#464)
+- chore(bq,sf,rs,pg): make remove drop functions instead of whole schema (#466)
+- fix(bq,sf,rs,pg|quadbin): improve precision of long lat conversion near the latitude limits (#461)
+- feat(bq,sf|transformations): add function ST_POINTONSURFACE (#469, #470)
+
 ## 2023-08-04
 
 - chore(bq|quadbin,h3): optimize quadbin/h3 polyfill performance (#421)

CONTRIBUTING.md

@@ -162,3 +162,68 @@ Images must be stored in `common/assets/`, and included with the following synta
 ```md
 ![My image](my_image.png)
 ```
+
+## Naming requirements
+
+The format of function and procedure names in the code must follow some conventions, so that the scripts that build the SQL code can determine the dependencies between them.
+
+### BigQuery
+
+Function/procedure invocations: the name must be quoted with backticks, including the project/dataset name. The opening bracket must follow the closing quote with no space between them:
+
+```sql
+SELECT `@@BQ_DATASET@@.A_FUNCTION`();
+CALL `@@BQ_DATASET@@.A_PROCEDURE`();
+```
+
+Function/procedure definitions: must be quoted as invocations, but parentheses should be on a separate line:
+
+```sql
+CREATE OR REPLACE FUNCTION `@@BQ_DATASET@@.A_FUNCTION`
+(
+    an_argument STRING
+)
+...
+```
+
+### Other clouds (Snowflake, Redshift, Postgres)
+
+Function/procedure invocations: The opening bracket must follow name with no space between them:
+
+
+```sql
+SELECT @@XX_SCHEMA@@.A_FUNCTION();
+CALL @@XX_SCHEMA@@.A_PROCEDURE();
+```
+
+Function/procedure definitions: must be quoted as invocations, but parentheses should be on a separate line:
+
+```sql
+CREATE OR REPLACE FUNCTION @@XX_SCHEMA@@.A_FUNCTION
+(
+  an_argument STRING
+)
+RETURNS STRING
+IMMUTABLE
+...
+```
+
+### Extra dependencies
+
+Sometimes the name of an invoked function or procedure is dynamically generated in the SQL code;
+in those cases it's necessary to add a coment in the file referencing all the functions/procedures that can potentially be called.
+
+In this comment, the names must appear as in actual invocations, following the rules of each cloud, i.e. including the argument parentheses, and
+in the case of BigQuery, the backtick quotes. For example:
+
+```sql
+/*
+Extra dependencies:
+`@@BQ_DATASET@@.SOME_FUNCTION`()
+*/
+```
+
+## Known Limitations
+### Snowflake
+
+Due to Snowflake Native Apps limitations at this moment TEMPORARY tables cannot be used within procedures. For the time being create normal tables and ensure that they are dropped.

clouds/bigquery/CHANGELOG.md

@@ -4,6 +4,16 @@ CARTO Analytics Toolbox Core for BigQuery.
 
 All notable commits to this project will be documented in this file.
 
+## [1.2.0] - 2024-01-17
+
+- chore: increase tests timeout to 200000 (#455)
+- feat(quadbin): add function QUADBIN_DISTANCE (#457)
+- fix(h3): fix broken reference in H3_POLYFILL_TABLE (#458, #460)
+- chore: fix naming dedicated deployments for releases (#462)
+- chore: make remove drop functions instead of whole schema (#466)
+- fix(quadbin): improve precision of long lat conversion near the latitude limits (#461)
+- feat(transformations): add function ST_POINTONSURFACE (#469)
+
 ## [1.1.0] - 2023-08-04
 
 - chore(quadbin,h3): optimize quadbin/h3 polyfill performance (#421)

clouds/bigquery/common/build_modules.js

@@ -99,7 +99,7 @@ functionsFilter.forEach(f => {
 if (!nodeps) {
     functions.forEach(mainFunction => {
         functions.forEach(depFunction => {
-            if (mainFunction.name != depFunction.name && depFunction.name !== 'SETUP') {
+            if (mainFunction.name != depFunction.name) {
                 const depFunctionMatches = [];
                 depFunctionMatches.push(...depFunction.content.replace(/(\r\n|\n|\r)/gm,' ').matchAll(new RegExp('(?<=(?<!TEMP )FUNCTION)(.*?)(?=AS |RETURNS)','g')));
                 depFunctionMatches.push(...depFunction.content.replace(/(\r\n|\n|\r)/gm,' ').matchAll(new RegExp('(?<=PROCEDURE)(.*?)(?=BEGIN)','g')));
@@ -109,7 +109,7 @@ if (!nodeps) {
                     qualifiedDepFunctName = qualifiedDepFunctName.split('.');
                     depFunctionNames.push(qualifiedDepFunctName[qualifiedDepFunctName.length - 1]);
                 })
-                if (depFunctionNames.some((depFunctionName) => mainFunction.content.includes(`DATASET@@.${depFunctionName}`))) {
+                if (depFunctionNames.some((depFunctionName) => mainFunction.content.includes(`DATASET@@.${depFunctionName}\`(`))) {
                     mainFunction.dependencies.push(depFunction.name);
                 }
             }

clouds/bigquery/common/package.json

@@ -1,7 +1,7 @@
 {
   "license": "BSD-3-Clause",
   "devDependencies": {
-    "@google-cloud/bigquery": "^5.3.0",
+    "@google-cloud/bigquery": "^7.3.0",
     "@rollup/plugin-commonjs": "^17.1.0",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^13.0.0",

clouds/bigquery/common/run-query.js

@@ -7,8 +7,7 @@ const BQ_PROJECT = process.env.BQ_PROJECT;
 const client = new BigQuery({ projectId: `${BQ_PROJECT}` });
 
 function runQuery (query) {
-    const query_options = { 'timeoutMs' : 120000 };
-    client.query(query, query_options);
+    client.query(query, { timeoutMs : 120000 });
 }
 
 const query = process.argv[2];

clouds/bigquery/common/run-script.js

@@ -14,21 +14,36 @@ const options = {
 };
 const bar = new cliProgress.SingleBar(options, cliProgress.Presets.shades_classic);
 
-const client = new BigQuery({ projectId: `${BQ_PROJECT}`, timeout: 600000 });
+const client = new BigQuery({ projectId: `${BQ_PROJECT}` });
+
+function apply_replacements (text) {
+    if (process.env.REPLACEMENTS) {
+        const replacements = process.env.REPLACEMENTS.split(' ');
+        for (let replacement of replacements) {
+            if (replacement) {
+                const pattern = new RegExp(`@@${replacement}@@`, 'g');
+                text = text.replace(pattern, process.env[replacement]);
+            }
+        }
+    }
+    return text;
+}
 
 async function runQueries (queries) {
-    const query_options = { 'timeoutMs' : 600000 };
     const n = queries.length;
     bar.start(n, 0);
     for (let i = 0; i < n; i++) {
-        let query = queries[i];
+        let query = apply_replacements(queries[i]);
 
         const pattern = /(FUNCTION|PROCEDURE)\s+(.*?)[(\n]/g;
         const results = pattern.exec(query);
         const result = results && results.reverse()[0]
         sqlFunction = result && result.split('.').reverse()[0]
 
-        await client.query(query, query_options);
+        await client.query({
+            query,
+            jobTimeoutMs: 600000
+        });
         bar.increment();
     }
     bar.stop(n);

clouds/bigquery/common/test-utils.js

@@ -10,7 +10,7 @@ const BQ_DATASET = process.env.BQ_DATASET;
 const client = new BigQuery({ projectId: `${BQ_PROJECT}` });
 
 async function runQuery (query, options) {
-    options = Object.assign({}, { 'timeoutMs' : 120000 }, options);
+    options = Object.assign({}, { timeoutMs : 120000 }, options);
     query = replaceBQPrefix(query);
     const [rows] = await client.query(query, options);
     return rows;

clouds/bigquery/libraries/javascript/src/transformations.js

@@ -11,6 +11,7 @@ import {
     along,
     lineString,
     cleanCoords,
+    pointOnFeature,
     multiPoint,
     point
 } from '@turf/turf';
@@ -28,6 +29,7 @@ export default {
     along,
     lineString,
     cleanCoords,
+    pointOnFeature,
     multiPoint,
     point
 };

clouds/bigquery/modules/Makefile

@@ -85,7 +85,7 @@ test: check $(NODE_MODULES_DEV)
 		if [ ! -z "$$TESTS" ]; then \
 			GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \
 			PATH="$(NODE_MODULES_DEV)/.bin/:$(PATH)" \
-			jest --testTimeout=150000 $(BAIL) --verbose --slowTestThreshold=20 --maxConcurrency=10 $$TESTS \
+			jest --testTimeout=200000 $(BAIL) --verbose --slowTestThreshold=20 --maxConcurrency=10 $$TESTS \
 			     --setupFilesAfterEnv "$(COMMON_DIR)/test-extend.js" || exit 1; \
 			OLD_TEST=$(TEST_DIR)/$$m/old-test; \
 			if [ -d $$OLD_TEST ]; then \
@@ -97,7 +97,9 @@ test: check $(NODE_MODULES_DEV)
 
 remove: check
 	echo "Removing modules..."
-	$(BQ) rm -r -f -d $(BQ_DEPLOY_DATASET)
+	REPLACEMENTS=$(REPLACEMENTS)" "$(REPLACEMENTS_EXTRA) \
+	GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \
+	$(COMMON_DIR)/run-script.js $(COMMON_DIR)/DROP_FUNCTIONS.sql
 
 clean:
 	echo "Cleaning modules..."