Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/DBSQLSession.ts
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ export default class DBSQLSession implements IDBSQLSession {
}

if (ProtocolVersion.supportsArrowCompression(this.serverProtocolVersion) && request.canDownloadResult !== true) {
request.canDecompressLZ4Result = (options.useLZ4Compression ?? clientConfig.useLZ4Compression) && Boolean(LZ4);
request.canDecompressLZ4Result = (options.useLZ4Compression ?? clientConfig.useLZ4Compression) && Boolean(LZ4());
}

const operationPromise = driver.executeStatement(request);
Expand Down
4 changes: 2 additions & 2 deletions lib/result/ArrowResultHandler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export default class ArrowResultHandler implements IResultsProvider<ArrowBatch>
this.arrowSchema = arrowSchema ?? hiveSchemaToArrowSchema(schema);
this.isLZ4Compressed = lz4Compressed ?? false;

if (this.isLZ4Compressed && !LZ4) {
if (this.isLZ4Compressed && !LZ4()) {
throw new HiveDriverError('Cannot handle LZ4 compressed result: module `lz4` not installed');
}
}
Expand All @@ -52,7 +52,7 @@ export default class ArrowResultHandler implements IResultsProvider<ArrowBatch>
let totalRowCount = 0;
rowSet?.arrowBatches?.forEach(({ batch, rowCount }) => {
if (batch) {
batches.push(this.isLZ4Compressed ? LZ4!.decode(batch) : batch);
batches.push(this.isLZ4Compressed ? LZ4()!.decode(batch) : batch);
totalRowCount += rowCount.toNumber(true);
}
});
Expand Down
4 changes: 2 additions & 2 deletions lib/result/CloudFetchResultHandler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
this.source = source;
this.isLZ4Compressed = lz4Compressed ?? false;

if (this.isLZ4Compressed && !LZ4) {
if (this.isLZ4Compressed && !LZ4()) {
throw new HiveDriverError('Cannot handle LZ4 compressed result: module `lz4` not installed');
}
}
Expand Down Expand Up @@ -64,7 +64,7 @@ export default class CloudFetchResultHandler implements IResultsProvider<ArrowBa
}

if (this.isLZ4Compressed) {
batch.batches = batch.batches.map((buffer) => LZ4!.decode(buffer));
batch.batches = batch.batches.map((buffer) => LZ4()!.decode(buffer));
}
return batch;
}
Expand Down
15 changes: 14 additions & 1 deletion lib/utils/lz4.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ function tryLoadLZ4Module(): LZ4Module | undefined {
return require('lz4'); // eslint-disable-line global-require
} catch (err) {
if (!(err instanceof Error) || !('code' in err)) {
// eslint-disable-next-line no-console
console.warn('Unexpected error loading LZ4 module: Invalid error object', err);
return undefined;
}
Expand All @@ -16,14 +17,26 @@ function tryLoadLZ4Module(): LZ4Module | undefined {
}

if (err.code === 'ERR_DLOPEN_FAILED') {
// eslint-disable-next-line no-console
console.warn('LZ4 native module failed to load: Architecture or version mismatch', err);
return undefined;
}

// If it's not a known error, return undefined
// eslint-disable-next-line no-console
console.warn('Unknown error loading LZ4 module: Unhandled error code', err);
return undefined;
}
}

export default tryLoadLZ4Module();
// The null is already tried resolving that failed
let resolvedModule: LZ4Module | null | undefined;

function getResolvedModule() {
if (resolvedModule === undefined) {
resolvedModule = tryLoadLZ4Module() ?? null;
}
return resolvedModule === null ? undefined : resolvedModule;
}

export default getResolvedModule;
156 changes: 156 additions & 0 deletions tests/unit/utils/lz4.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
import { expect } from 'chai';
import sinon from 'sinon';

describe('lz4 module loader', () => {
let moduleLoadStub: sinon.SinonStub | undefined;
let consoleWarnStub: sinon.SinonStub;

beforeEach(() => {
consoleWarnStub = sinon.stub(console, 'warn');
});

afterEach(() => {
consoleWarnStub.restore();
if (moduleLoadStub) {
moduleLoadStub.restore();
}
// Clear module cache
Object.keys(require.cache).forEach((key) => {
if (key.includes('lz4')) {
delete require.cache[key];
}
});
});

const mockModuleLoad = (lz4MockOrError: unknown): { restore: () => void; wasLz4LoadAttempted: () => boolean } => {
// eslint-disable-next-line global-require
const Module = require('module');
const originalLoad = Module._load;
let lz4LoadAttempted = false;

Module._load = (request: string, parent: unknown, isMain: boolean) => {
if (request === 'lz4') {
lz4LoadAttempted = true;
if (lz4MockOrError instanceof Error) {
throw lz4MockOrError;
}
return lz4MockOrError;
}
return originalLoad.call(Module, request, parent, isMain);
};

return {
restore: () => {
Module._load = originalLoad;
},
wasLz4LoadAttempted: () => lz4LoadAttempted,
};
};

const loadLz4Module = () => {
delete require.cache[require.resolve('../../../lib/utils/lz4')];
// eslint-disable-next-line global-require
return require('../../../lib/utils/lz4');
};

it('should successfully load and use lz4 module when available', () => {
const fakeLz4 = {
encode: (buf: Buffer) => {
const compressed = Buffer.from(`compressed:${buf.toString()}`);
return compressed;
},
decode: (buf: Buffer) => {
const decompressed = buf.toString().replace('compressed:', '');
return Buffer.from(decompressed);
},
};

const { restore } = mockModuleLoad(fakeLz4);
const moduleExports = loadLz4Module();
const lz4Module = moduleExports.default();
restore();

expect(lz4Module).to.not.be.undefined;
expect(lz4Module.encode).to.be.a('function');
expect(lz4Module.decode).to.be.a('function');

const testData = Buffer.from('Hello, World!');
const compressed = lz4Module.encode(testData);
const decompressed = lz4Module.decode(compressed);

expect(decompressed.toString()).to.equal('Hello, World!');
expect(consoleWarnStub.called).to.be.false;
});

it('should return undefined when lz4 module fails to load with MODULE_NOT_FOUND', () => {
const err: NodeJS.ErrnoException = new Error("Cannot find module 'lz4'");
err.code = 'MODULE_NOT_FOUND';

const { restore } = mockModuleLoad(err);
const moduleExports = loadLz4Module();
const lz4Module = moduleExports.default();
restore();

expect(lz4Module).to.be.undefined;
expect(consoleWarnStub.called).to.be.false;
});

it('should return undefined and log warning when lz4 fails with ERR_DLOPEN_FAILED', () => {
const err: NodeJS.ErrnoException = new Error('Module did not self-register');
err.code = 'ERR_DLOPEN_FAILED';

const { restore } = mockModuleLoad(err);
const moduleExports = loadLz4Module();
const lz4Module = moduleExports.default();
restore();

expect(lz4Module).to.be.undefined;
expect(consoleWarnStub.calledOnce).to.be.true;
expect(consoleWarnStub.firstCall.args[0]).to.include('Architecture or version mismatch');
});

it('should return undefined and log warning when lz4 fails with unknown error code', () => {
const err: NodeJS.ErrnoException = new Error('Some unknown error');
err.code = 'UNKNOWN_ERROR';

const { restore } = mockModuleLoad(err);
const moduleExports = loadLz4Module();
const lz4Module = moduleExports.default();
restore();

expect(lz4Module).to.be.undefined;
expect(consoleWarnStub.calledOnce).to.be.true;
expect(consoleWarnStub.firstCall.args[0]).to.include('Unhandled error code');
});

it('should return undefined and log warning when error has no code property', () => {
const err = new Error('Error without code');

const { restore } = mockModuleLoad(err);
const moduleExports = loadLz4Module();
const lz4Module = moduleExports.default();
restore();

expect(lz4Module).to.be.undefined;
expect(consoleWarnStub.calledOnce).to.be.true;
expect(consoleWarnStub.firstCall.args[0]).to.include('Invalid error object');
});

it('should not attempt to load lz4 module when getResolvedModule is not called', () => {
const fakeLz4 = {
encode: () => Buffer.from(''),
decode: () => Buffer.from(''),
};

const { restore, wasLz4LoadAttempted } = mockModuleLoad(fakeLz4);

// Load the module but don't call getResolvedModule
loadLz4Module();
// Note: we're NOT calling .default() here

restore();

expect(wasLz4LoadAttempted()).to.be.false;
expect(consoleWarnStub.called).to.be.false;
});
});