Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
93cb5dd
[DO NOT MERGE] tiny diff to trigger tests
mcheshkov Oct 31, 2024
cf875b1
[DO NOT MERGE] Add debug dig and curl calls
mcheshkov Nov 1, 2024
fa9d6bc
[DO NOT MERGE] fix dig call
mcheshkov Nov 1, 2024
71f8df9
[DO NOT MERGE] more curl calls
mcheshkov Nov 1, 2024
bd08e77
[DO NOT MERGE] add debug calls from node
mcheshkov Nov 1, 2024
558de58
[DO NOT MERGE] add debug calls from node with token
mcheshkov Nov 2, 2024
4368ed2
[DO NOT MERGE] fix debug calls from node with token
mcheshkov Nov 2, 2024
4ab47ed
[DO NOT MERGE] add debug calls from node thru driver
mcheshkov Nov 2, 2024
dba6808
[DO NOT MERGE] tune debug calls from node thru driver
mcheshkov Nov 2, 2024
fc1274c
[DO NOT MERGE] tune debug calls from node thru driver
mcheshkov Nov 2, 2024
185c62c
[DO NOT MERGE] bump JDBC driver version
mcheshkov Nov 2, 2024
ca63c63
[DO NOT MERGE] some debug for JDBC
mcheshkov Nov 2, 2024
d1bc137
[DO NOT MERGE] more debug for mvn in JDBC
mcheshkov Nov 2, 2024
2f45478
[DO NOT MERGE] more debug for mvn in JDBC
mcheshkov Nov 2, 2024
cdef2e0
[DO NOT MERGE] more debug for jinst in JDBC
mcheshkov Nov 2, 2024
79fa08f
[DO NOT MERGE] more debug from node
mcheshkov Nov 2, 2024
553fdff
[DO NOT MERGE] simplify test script
mcheshkov Nov 2, 2024
1f3d7a7
[DO NOT MERGE] simplify test script
mcheshkov Nov 2, 2024
a4a9681
[DO NOT MERGE] more debug from JDBC
mcheshkov Nov 2, 2024
b34c26a
[DO NOT MERGE] more debug from JDBC
mcheshkov Nov 2, 2024
04d9dab
[DO NOT MERGE] more debug from JDBC
mcheshkov Nov 2, 2024
35a012e
[DO NOT MERGE] more debug from JDBC
mcheshkov Nov 2, 2024
fc8e331
[DO NOT MERGE] fix databricks accept
mcheshkov Nov 3, 2024
80137b3
[DO NOT MERGE] fix databricks test jdbc url schema
mcheshkov Nov 3, 2024
f3764fe
[DO NOT MERGE] add debug timeout
mcheshkov Nov 3, 2024
ade37ea
[DO NOT MERGE] add fake http path
mcheshkov Nov 3, 2024
a7f4664
[DO NOT MERGE] hide some logs and just run tests
mcheshkov Nov 3, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .github/workflows/drivers-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,11 @@ jobs:
retry_wait_seconds: 15
timeout_minutes: 20
command: |
dig ohio.cloud.databricks.com.
curl -v https://ohio.cloud.databricks.com
curl -v https://ohio.cloud.databricks.com/sql/1.0/warehouses/
CUBEJS_DB_DATABRICKS_ACCEPT_POLICY=true node --eval='require("https").get("https://ohio.cloud.databricks.com/sql/1.0/warehouses/", (res) => {console.log("res status and headers", res.statusCode, res.statusMessage, res.headers); res.on("data", (chunk) => console.log("res data", chunk.toString()))})'
cd ./packages/cubejs-testing-drivers
export DEBUG=testcontainers
yarn ${{ matrix.database }}-full
CUBEJS_DB_DATABRICKS_ACCEPT_POLICY=true yarn ${{ matrix.database }}-full

12 changes: 9 additions & 3 deletions packages/cubejs-databricks-jdbc-driver/src/installer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ import path from 'path';
import { downloadAndExtractFile, getEnv } from '@cubejs-backend/shared';

function acceptedByEnv() {
console.log('acceptedByEnv call');

const acceptStatus = getEnv('databrickAcceptPolicy');
console.log('acceptedByEnv call acceptStatus', acceptStatus);
if (acceptStatus) {
console.log('You accepted Terms & Conditions for JDBC driver from DataBricks by CUBEJS_DB_DATABRICKS_ACCEPT_POLICY');
}
Expand All @@ -16,20 +19,23 @@ function acceptedByEnv() {
}

export async function downloadJDBCDriver(): Promise<string | null> {
console.log('downloadJDBCDriver call');
const driverAccepted = acceptedByEnv();

console.log('downloadJDBCDriver call driverAccepted', driverAccepted);

if (driverAccepted) {
console.log('Downloading DatabricksJDBC42-2.6.29.1051');
console.log('Downloading DatabricksJDBC42-2.6.40.1071');

await downloadAndExtractFile(
'https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/DatabricksJDBC42-2.6.29.1051.zip',
'https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.40/DatabricksJDBC42-2.6.40.1071.zip',
{
showProgress: true,
cwd: path.resolve(path.join(__dirname, '..', 'download')),
}
);

console.log('Release notes: https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.29/docs/release-notes.txt');
console.log('Release notes: https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.40/docs/release-notes.txt');

return path.resolve(path.join(__dirname, '..', 'download', 'DatabricksJDBC42.jar'));
}
Expand Down
24 changes: 23 additions & 1 deletion packages/cubejs-jdbc-driver/src/JDBCDriver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import { DriverOptionsInterface, SupportedDrivers } from './supported-drivers';
// eslint-disable-next-line @typescript-eslint/no-unused-vars
import { JDBCDriverConfiguration } from './types';
import { QueryStream, nextFn, Row, transformRow } from './QueryStream';

Check warning on line 26 in packages/cubejs-jdbc-driver/src/JDBCDriver.ts

View workflow job for this annotation

GitHub Actions / lint

'Row' is defined but never used. Allowed unused vars must match /^_.*/u

/* eslint-disable no-restricted-syntax,import/no-extraneous-dependencies */
const DriverManager = require('@cubejs-backend/jdbc/lib/drivermanager');
Expand All @@ -37,14 +37,26 @@
const initMvn = (customClassPath: any) => {
if (!mvnPromise) {
mvnPromise = new Promise((resolve, reject) => {
console.log('new mvnPromise');

const options = {
packageJsonPath: `${path.join(__dirname, '../..')}/package.json`,
};
console.log('mvnPromise options', options);
mvn(options, (err: any, mvnResults: any) => {
console.log('mvnPromise callback err', err);
console.log('mvnPromise callback mvnResults', mvnResults);

console.log('mvnPromise callback err.message.includes', err.message.includes('Could not find java property'));

if (err && !err.message.includes('Could not find java property')) {
reject(err);
} else {
console.log('jinst.isJvmCreated()', jinst.isJvmCreated());

if (!jinst.isJvmCreated()) {
console.log('before jinst.addOption');

jinst.addOption('-Xrs');
jinst.addOption('-Dfile.encoding=UTF8');

Expand All @@ -53,6 +65,8 @@
jinst.addOption('--add-opens=java.base/java.nio=ALL-UNNAMED');

const classPath = (mvnResults && mvnResults.classpath || []).concat(customClassPath || []);

console.log('before jinst.setupClasspath');
jinst.setupClasspath(classPath);
}
resolve();
Expand Down Expand Up @@ -137,14 +151,18 @@

this.pool = genericPool.createPool({
create: async () => {
await initMvn(await this.getCustomClassPath());
// console.log('before await this.getCustomClassPath()', new Error().stack);
const ccp = await this.getCustomClassPath();
// console.log('before await initMvn(ccp)');
await initMvn(ccp);

if (!this.jdbcProps) {
/** @protected */
this.jdbcProps = this.getJdbcProperties();
}

const getConnection = promisify(DriverManager.getConnection.bind(DriverManager));
// console.log('new Connection this.config.url', this.config.url);
return new Connection(await getConnection(this.config.url, this.jdbcProps));
},
// @ts-expect-error Promise<Function> vs Promise<void>
Expand Down Expand Up @@ -183,6 +201,10 @@
acquireTimeoutMillis: 120000,
...(poolOptions || {})
}) as ExtendedPool;

this.pool.on('factoryCreateError', (err) => {
console.log('pool factoryCreateError', err);
});
}

protected async getCustomClassPath() {
Expand Down
2 changes: 1 addition & 1 deletion packages/cubejs-testing-drivers/src/tests/testQueries.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ export function testQueries(type: string, { includeIncrementalSchemaSuite, exten
}
console.log(`Creating ${queries.length} fixture tables completed`);
} catch (e: any) {
console.log('Error creating fixtures', e.stack);
console.log('Error creating TEST fixtures', e.stack);
throw e;
}
});
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,67 @@
// import * as https from 'node:https';
//
// import D from '@cubejs-backend/databricks-jdbc-driver';

import { testQueries } from '../src/tests/testQueries';

// https.get(
// 'https://ohio.cloud.databricks.com/sql/1.0/warehouses/',
// (res) => {
// console.log(
// 'res status and headers',
// res.statusCode,
// res.statusMessage,
// res.headers
// );
// res.on('data', (chunk) => console.log('res data', chunk.toString()));
// }
// );
//
// https.get(
// 'https://ohio.cloud.databricks.com/sql/1.0/warehouses/',
// {
// headers: {
// authorization: `Bearer ${process.env.DRIVERS_TESTS_CUBEJS_DB_DATABRICKS_TOKEN}`,
// }
// },
// (res) => {
// console.log(
// 'res with token status and headers',
// res.statusCode,
// res.statusMessage,
// res.headers
// );
// res.on('data', (chunk) => console.log('res with token data', chunk.toString()));
// }
// );
//
// async function f() {
// const source = new D({
// url: 'jdbc:databricks://ohio.cloud.databricks.com/default;httpPath=/sql/1.0/warehouses/;',
// maxPoolSize: 1,
// });
// console.log('test driver constructed');
// const qRes = await source.query('SELECT 1 as foo;', []);
// console.log('qRes', qRes);
// }
//
// f()
// .then(
// () => {
// console.log('driver test query succ');
// process.exit(0);
// },
// e => {
// console.log('driver test query failed', e);
// process.exit(1);
// }
// );
//
// setTimeout(() => {
// console.log('TIMING OUT');
// process.exit(1);
// }, 3 * 60 * 1000);

testQueries('databricks-jdbc', {
includeIncrementalSchemaSuite: true,
includeHLLSuite: true,
Expand Down
Loading