From 64adc9851b20faa381550747c8e6d863ef6ba406 Mon Sep 17 00:00:00 2001 From: inDream Date: Thu, 21 Jul 2016 21:19:32 +0800 Subject: [PATCH 01/10] Init commit --- lib/dialects/abstract/query-generator.js | 46 ++++++++++++- lib/dialects/postgres/index.js | 1 + lib/dialects/postgres/query-generator.js | 85 +++++++++++++++++++----- lib/dialects/postgres/query.js | 8 ++- test/integration/model/upsert.test.js | 25 ++++--- 5 files changed, 137 insertions(+), 28 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 087d4fe95926..eb1af9beb5bc 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -133,6 +133,22 @@ class QueryGenerator { emptyQuery += ' VALUES ()'; } + if (this.dialect === 'postgres') { + if (options.onConflict && semver.gte(this.sequelize.options.databaseVersion, '9.5.0')) { + const constraint = options.conflict.constraint; + const target = options.conflict.target; + if (constraint) { + valueQuery += ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint) + options.onConflict; + } else if (target) { + const conflictTarget = target.map(e => this.quoteIdentifier(e)).join(', '); + valueQuery += ' ON CONFLICT (' + conflictTarget + ') ' + options.onConflict; + } + valueQuery += ' RETURNING ' + options.returning; + } else { + options.onConflictFallback = true; + } + } + if (this._dialect.supports.returnValues && options.returning) { if (this._dialect.supports.returnValues.returning) { valueQuery += ' RETURNING *'; @@ -274,11 +290,12 @@ class QueryGenerator { options = options || {}; fieldMappedAttributes = fieldMappedAttributes || {}; - const query = 'INSERT<%= ignoreDuplicates %> INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %><%= onDuplicateKeyUpdate %><%= onConflictDoNothing %><%= returning %>;'; + const query = 'INSERT<%= ignoreDuplicates %> INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %><%= onDuplicateKeyUpdate %><%= onConflictDoNothing %><%= onConflictKeyUpdate %><%= returning %>;'; const tuples = []; const serials = {}; const allAttributes = []; let onDuplicateKeyUpdate = ''; + let onConflictKeyUpdate = ''; for (const fieldValueHash of fieldValueHashes) { _.forOwn(fieldValueHash, (value, key) => { @@ -316,12 +333,39 @@ class QueryGenerator { }).join(','); } + if (this._dialect.supports.updateOnConflict && options.updateOnConflict) { + options.updateOnConflict = Utils._.extend({ + target: 'id', + update: attrValueHashes.map(value => value.name).filter(value => value !== 'id') + }, options.updateOnConflict || {}); + + let conflict = ''; + const constraint = this.quoteIdentifier(options.updateOnConflict.constraint); + const target = this.quoteIdentifier(options.updateOnConflict.target); + if (constraint) { + conflict = ' ON CONFLICT ON CONSTRAINT ' + target; + } else { + conflict = ' ON CONFLICT (' + target + ')'; + } + + if (options.updateOnConflict.update === false) { + onConflictKeyUpdate += ' DO NOTHING '; + } else { + onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + options.updateOnConflict.update.map(attr => { + const field = rawAttributes && rawAttributes[attr] && rawAttributes[attr].field || attr; + const key = this.quoteIdentifier(field); + return key + ' = EXCLUDED.' + key; + }).join(','); + } + } + const replacements = { ignoreDuplicates: options.ignoreDuplicates ? this._dialect.supports.inserts.ignoreDuplicates : '', table: this.quoteTable(tableName), attributes: allAttributes.map(attr => this.quoteIdentifier(attr)).join(','), tuples: tuples.join(','), onDuplicateKeyUpdate, + onConflictKeyUpdate, returning: this._dialect.supports.returnValues && options.returning ? ' RETURNING *' : '', onConflictDoNothing: options.ignoreDuplicates ? this._dialect.supports.inserts.onConflictDoNothing : '' }; diff --git a/lib/dialects/postgres/index.js b/lib/dialects/postgres/index.js index e1b65df57aac..48e19d303da4 100644 --- a/lib/dialects/postgres/index.js +++ b/lib/dialects/postgres/index.js @@ -53,6 +53,7 @@ PostgresDialect.prototype.supports = _.merge(_.cloneDeep(AbstractDialect.prototy JSONB: true, HSTORE: true, deferrableConstraints: true, + updateOnConflict: true, searchPath: true }); diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index e1f1d239e975..47c28929e048 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -347,22 +347,75 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { } upsertQuery(tableName, insertValues, updateValues, where, model, options) { - const primaryField = this.quoteIdentifier(model.primaryKeyField); - - const upsertOptions = _.defaults({ bindParam: false }, options); - const insert = this.insertQuery(tableName, insertValues, model.rawAttributes, upsertOptions); - const update = this.updateQuery(tableName, updateValues, where, upsertOptions, model.rawAttributes); - - insert.query = insert.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); - update.query = update.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); - - return this.exceptionFn( - 'sequelize_upsert', - tableName, - 'OUT created boolean, OUT primary_key text', - `${insert.query} created := true;`, - `${update.query}; created := false` - ); + const rawAttributes = model.rawAttributes; + const indexes = model.options.indexes; + const uniqueKeys = Object.keys(rawAttributes).filter(e => rawAttributes[e].unique); + const uniqueIndexes = indexes.filter(e => e.unique).map(e => e.name); + const uniqueCount = uniqueKeys.length + uniqueIndexes.length; + if (uniqueCount > 1 && (!options.conflict || options.conflict && !options.conflict.constraint) || + semver.lt(this.sequelize.options.databaseVersion, '9.5.0')) { + options.onConflictFallback = true; + const primaryField = this.quoteIdentifier(model.primaryKeyField); + + const upsertOptions = _.defaults({ bindParam: false }, options); + const insert = this.insertQuery(tableName, insertValues, model.rawAttributes, upsertOptions); + const update = this.updateQuery(tableName, updateValues, where, upsertOptions, model.rawAttributes); + + insert.query = insert.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); + update.query = update.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); + + return this.exceptionFn( + 'sequelize_upsert', + tableName, + 'OUT created boolean, OUT primary_key text', + `${insert.query} created := true;`, + `${update.query}; created := false` + ); + } + + if (!options.conflict) { + let target = 'id'; + for (const key of Object.keys(rawAttributes)) { + if (rawAttributes[key].primaryKey) { + target = rawAttributes[key].field; + } + } + options.conflict = { + target, + update: Object.keys(updateValues).filter(value => value !== target) + }; + } + + if (options.conflict.update === false) { + options.onConflict = ' DO NOTHING'; + } else { + options.conflict.target = options.conflict.target || uniqueKeys; + if (!Array.isArray(options.conflict.target)) { + options.conflict.target = [options.conflict.target]; + } + if (!options.conflict.update) { + options.conflict.update = Object.keys(updateValues) + .filter(value => options.conflict.target.indexOf(value) === -1); + } + + if (rawAttributes.updatedAt) { + insertValues.updatedAt = new Date(); + if (options.conflict.update.indexOf('updatedAt') === -1) { + options.conflict.update.push('updatedAt'); + } + } + + options.onConflict = 'DO UPDATE SET ' + options.conflict.update.map(key => { + key = this.quoteIdentifier(key); + return key + ' = EXCLUDED.' + key; + }).join(', '); + } + + if (options.returning === undefined) { + options.returning = '*'; + } + + return this.insertQuery(tableName, insertValues, rawAttributes, options); } truncateTableQuery(tableName, options = {}) { diff --git a/lib/dialects/postgres/query.js b/lib/dialects/postgres/query.js index 03117e96900a..b5c753115e1e 100644 --- a/lib/dialects/postgres/query.js +++ b/lib/dialects/postgres/query.js @@ -249,7 +249,13 @@ class Query extends AbstractQuery { } else if (QueryTypes.BULKDELETE === this.options.type) { return parseInt(rowCount, 10); } else if (this.isUpsertQuery()) { - return rows[0]; + if (this.options.onConflictFallback) { + return rows[0]; + } else { + const item = this.options.plain ? rows[0] : + this.options.instance.build(rows[0]); + return this.options.returning === true ? item : rows.length; + } } else if (this.isInsertQuery() || this.isUpdateQuery()) { if (this.instance && this.instance.dataValues) { for (const key in rows[0]) { diff --git a/test/integration/model/upsert.test.js b/test/integration/model/upsert.test.js index bcb4ed369afb..b4b973b5fcec 100644 --- a/test/integration/model/upsert.test.js +++ b/test/integration/model/upsert.test.js @@ -86,7 +86,8 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); it('works with upsert on a composite key', function() { - return this.User.upsert({ foo: 'baz', bar: 19, username: 'john' }).bind(this).then(function(created) { + const options = { conflict: { constraint: 'users_foo_bar_key' } }; + return this.User.upsert({ foo: 'baz', bar: 19, username: 'john' }, options).bind(this).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; } else { @@ -94,7 +95,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { } this.clock.tick(1000); - return this.User.upsert({ foo: 'baz', bar: 19, username: 'doe' }); + return this.User.upsert({ foo: 'baz', bar: 19, username: 'doe' }, options); }).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -142,12 +143,13 @@ describe(Support.getTestDialectTeaser('Model'), () => { }, username: DataTypes.STRING }); + const options = { conflict: { constraint: 'users_pkey' } }; return User.sync({ force: true }).bind(this).then(() => { return Promise.all([ // Create two users - User.upsert({ a: 'a', b: 'b', username: 'john' }), - User.upsert({ a: 'a', b: 'a', username: 'curt' }) + User.upsert({ a: 'a', b: 'b', username: 'john' }, options), + User.upsert({ a: 'a', b: 'a', username: 'curt' }, options), ]); }).spread(function(created1, created2) { if (dialect === 'sqlite') { @@ -160,7 +162,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { this.clock.tick(1000); // Update the first one - return User.upsert({ a: 'a', b: 'b', username: 'doe' }); + return User.upsert({ a: 'a', b: 'b', username: 'doe' }, options); }).then(created => { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -268,7 +270,8 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); it('works with primary key using .field', function() { - return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'first' }).bind(this).then(function(created) { + const options = { conflict: { target: 'userId' } }; + return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'first' }, options).bind(this).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; } else { @@ -276,7 +279,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { } this.clock.tick(1000); - return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'second' }); + return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'second' }, options); }).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -464,14 +467,15 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); return User.sync({ force: true }).then(() => { - return User.upsert({ name: 'user1', address: 'address', city: 'City' }) + const options = { conflict: { target: ['name', 'address'] } }; + return User.upsert({ name: 'user1', address: 'address', city: 'City' }, options) .then(created => { if (dialect === 'sqlite') { expect(created).to.be.undefined; } else { expect(created).to.be.ok; } - return User.upsert({ name: 'user1', address: 'address', city: 'New City' }); + return User.upsert({ name: 'user1', address: 'address', city: 'New City' }, options); }).then(created => { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -537,7 +541,8 @@ describe(Support.getTestDialectTeaser('Model'), () => { // this record is soft deleted User.create({ name: 'user3', deletedAt: -Infinity }) ]).then(() => { - return User.upsert({ name: 'user1', address: 'address' }); + const options = { conflict: { target: ['name', 'deletedAt'] } }; + return User.upsert({ name: 'user1', address: 'address' }, options); }).then(() => { return User.findAll({ where: { address: null } From 3dfdd7c3da6a5f978ea31f02785cf7257ea3afa5 Mon Sep 17 00:00:00 2001 From: inDream Date: Tue, 10 Oct 2017 14:13:18 +0800 Subject: [PATCH 02/10] Handle postgres version 10 --- lib/dialects/abstract/connection-manager.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/dialects/abstract/connection-manager.js b/lib/dialects/abstract/connection-manager.js index 45040691aa9b..1aba627e3c91 100644 --- a/lib/dialects/abstract/connection-manager.js +++ b/lib/dialects/abstract/connection-manager.js @@ -255,6 +255,10 @@ class ConnectionManager { _options.logging.__testLoggingFn = true; return this.sequelize.databaseVersion(_options).then(version => { + if (this.dialectName === 'postgres' && _.isString(version) && + !version.match(/\.\d+\./)) { + version += '.0'; + } this.sequelize.options.databaseVersion = semver.valid(version) ? version : this.defaultVersion; this.versionPromise = null; From fba14a72f0d8d68c336b00aed1f37cc7f1052298 Mon Sep 17 00:00:00 2001 From: inDream Date: Sun, 28 Jan 2018 23:18:20 +0800 Subject: [PATCH 03/10] Update for new return values --- lib/dialects/abstract/query-generator.js | 2 +- lib/dialects/postgres/query-generator.js | 31 ++++++++++++++---------- lib/dialects/postgres/query.js | 6 ++--- lib/model.js | 3 +++ lib/query-interface.js | 4 +-- 5 files changed, 27 insertions(+), 19 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index eb1af9beb5bc..bf5267c90ca7 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -149,7 +149,7 @@ class QueryGenerator { } } - if (this._dialect.supports.returnValues && options.returning) { + if (this._dialect.supports.returnValues && options.returning && !options.isPgUpsert) { if (this._dialect.supports.returnValues.returning) { valueQuery += ' RETURNING *'; emptyQuery += ' RETURNING *'; diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index 47c28929e048..b5f38cd3cec1 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -350,8 +350,17 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { const rawAttributes = model.rawAttributes; const indexes = model.options.indexes; const uniqueKeys = Object.keys(rawAttributes).filter(e => rawAttributes[e].unique); + if (!uniqueKeys.length) { + uniqueKeys.push(model.primaryKeyField); + } const uniqueIndexes = indexes.filter(e => e.unique).map(e => e.name); const uniqueCount = uniqueKeys.length + uniqueIndexes.length; + const fields = Object.keys(model.fieldAttributeMap); + const fieldMap = {}; + fields.forEach(field => { + fieldMap[model.fieldAttributeMap[field]] = field; + }); + if (uniqueCount > 1 && (!options.conflict || options.conflict && !options.conflict.constraint) || semver.lt(this.sequelize.options.databaseVersion, '9.5.0')) { options.onConflictFallback = true; @@ -373,16 +382,14 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { ); } + const getUpdateArray = (values, target) => + Object.keys(values).map(key => fieldMap[key] || key).filter(value => target.indexOf(value) === -1); + + options.isPgUpsert = true; if (!options.conflict) { - let target = 'id'; - for (const key of Object.keys(rawAttributes)) { - if (rawAttributes[key].primaryKey) { - target = rawAttributes[key].field; - } - } options.conflict = { - target, - update: Object.keys(updateValues).filter(value => value !== target) + target: uniqueKeys, + update: getUpdateArray(updateValues, uniqueKeys) }; } @@ -393,9 +400,9 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { if (!Array.isArray(options.conflict.target)) { options.conflict.target = [options.conflict.target]; } + options.conflict.target = options.conflict.target.map(key => fieldMap[key] || key); if (!options.conflict.update) { - options.conflict.update = Object.keys(updateValues) - .filter(value => options.conflict.target.indexOf(value) === -1); + options.conflict.update = getUpdateArray(updateValues, options.conflict.target); } if (rawAttributes.updatedAt) { @@ -411,9 +418,7 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { }).join(', '); } - if (options.returning === undefined) { - options.returning = '*'; - } + options.returning = '*, (xmax = 0)::bool AS xmax'; return this.insertQuery(tableName, insertValues, rawAttributes, options); } diff --git a/lib/dialects/postgres/query.js b/lib/dialects/postgres/query.js index b5c753115e1e..abc452b747be 100644 --- a/lib/dialects/postgres/query.js +++ b/lib/dialects/postgres/query.js @@ -252,9 +252,9 @@ class Query extends AbstractQuery { if (this.options.onConflictFallback) { return rows[0]; } else { - const item = this.options.plain ? rows[0] : - this.options.instance.build(rows[0]); - return this.options.returning === true ? item : rows.length; + this.options.fieldMap = this.options.model.fieldAttributeMap; + const items = this.options.plain ? rows : this.handleSelectQuery(rows); + return [items[0], rows[0].xmax]; } } else if (this.isInsertQuery() || this.isUpdateQuery()) { if (this.instance && this.instance.dataValues) { diff --git a/lib/model.js b/lib/model.js index 756db5d5084e..b659be377415 100644 --- a/lib/model.js +++ b/lib/model.js @@ -2452,6 +2452,9 @@ class Model { }).then(() => { return this.QueryInterface.upsert(this.getTableName(options), insertValues, updateValues, instance.where(), this, options); }).spread((created, primaryKey) => { + if (primaryKey === true || primaryKey === false) { + return options.returning === true ? [created, primaryKey] : primaryKey; + } if (options.returning === true && primaryKey) { return this.findById(primaryKey, options).then(record => [record, created]); } diff --git a/lib/query-interface.js b/lib/query-interface.js index 639b7ee945ad..f0aaa393620c 100644 --- a/lib/query-interface.js +++ b/lib/query-interface.js @@ -895,13 +895,13 @@ class QueryInterface { where = { [Op.or]: wheres }; options.type = QueryTypes.UPSERT; - options.raw = true; + options.raw = options.isPgUpsert || false; const sql = this.QueryGenerator.upsertQuery(tableName, insertValues, updateValues, where, model, options); return this.sequelize.query(sql, options).then(result => { switch (this.sequelize.options.dialect) { case 'postgres': - return [result.created, result.primary_key]; + return options.isPgUpsert ? result : [result.created, result.primary_key]; case 'mssql': return [ From 2bf949ac18987a690f6d31a0cf2a0b6fb062b4e9 Mon Sep 17 00:00:00 2001 From: inDream Date: Thu, 12 Apr 2018 03:15:32 +0800 Subject: [PATCH 04/10] Fix bulkCreate and add test --- lib/dialects/abstract/query-generator.js | 21 +++++++++-------- test/integration/model/upsert.test.js | 29 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index bf5267c90ca7..12f5e22c0071 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -334,25 +334,26 @@ class QueryGenerator { } if (this._dialect.supports.updateOnConflict && options.updateOnConflict) { - options.updateOnConflict = Utils._.extend({ - target: 'id', - update: attrValueHashes.map(value => value.name).filter(value => value !== 'id') + const { primaryKeyField, rawAttributes } = options.model; + options.updateOnConflict = _.extend({ + constraint: '', + target: primaryKeyField, + update: allAttributes.filter(value => value !== primaryKeyField) }, options.updateOnConflict || {}); + const { constraint, target, update } = options.updateOnConflict; let conflict = ''; - const constraint = this.quoteIdentifier(options.updateOnConflict.constraint); - const target = this.quoteIdentifier(options.updateOnConflict.target); if (constraint) { - conflict = ' ON CONFLICT ON CONSTRAINT ' + target; + conflict = ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint); } else { - conflict = ' ON CONFLICT (' + target + ')'; + conflict = ' ON CONFLICT (' + this.quoteIdentifier(target) + ')'; } - if (options.updateOnConflict.update === false) { + if (update === false) { onConflictKeyUpdate += ' DO NOTHING '; } else { - onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + options.updateOnConflict.update.map(attr => { - const field = rawAttributes && rawAttributes[attr] && rawAttributes[attr].field || attr; + onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + update.map(attr => { + const field = rawAttributes[attr] ? rawAttributes[attr].fieldName : attr; const key = this.quoteIdentifier(field); return key + ' = EXCLUDED.' + key; }).join(','); diff --git a/test/integration/model/upsert.test.js b/test/integration/model/upsert.test.js index b4b973b5fcec..25293b42785b 100644 --- a/test/integration/model/upsert.test.js +++ b/test/integration/model/upsert.test.js @@ -552,6 +552,35 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); }); }); + + it('works with bulkCreate', function() { + const User = this.sequelize.define('User', { + name: { + type: DataTypes.STRING, + primaryKey: true + }, + address: DataTypes.STRING + }); + + return User.sync({ force: true }).then(() => { + return User.bulkCreate([ + { name: 'user1', address: 'user1' }, + { name: 'user2' } + ]).then(() => { + return User.bulkCreate([ + { name: 'user1' }, + { name: 'user2', address: 'user2' } + ], { updateOnConflict: true }); + }).then(() => { + return User.findAll({ + where: { address: null } + }); + }).then(users => { + expect(users).to.have.lengthOf(1); + expect(users[0].name).to.equal('user1'); + }); + }); + }); } if (current.dialect.supports.returnValues) { From 3735c5c23dee96000eeecaf94949aee7aaeeb0d9 Mon Sep 17 00:00:00 2001 From: Noah Prince Date: Fri, 22 Jun 2018 17:41:59 -0500 Subject: [PATCH 05/10] feat(query-generator): Allow index predicates in postgres upsert. --- lib/dialects/abstract/query-generator.js | 26 +++++++++++++++++++----- lib/dialects/postgres/query-generator.js | 20 ++++++++++-------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 12f5e22c0071..cb3e589e1ec3 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -136,12 +136,19 @@ class QueryGenerator { if (this.dialect === 'postgres') { if (options.onConflict && semver.gte(this.sequelize.options.databaseVersion, '9.5.0')) { const constraint = options.conflict.constraint; - const target = options.conflict.target; + const target = Array.isArray(options.conflict.target) ? options.conflict.target : [options.conflict.target]; if (constraint) { valueQuery += ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint) + options.onConflict; } else if (target) { const conflictTarget = target.map(e => this.quoteIdentifier(e)).join(', '); - valueQuery += ' ON CONFLICT (' + conflictTarget + ') ' + options.onConflict; + valueQuery += ' ON CONFLICT (' + conflictTarget + ') '; + + const conflictPredicate = options.conflict.update.where; + if (conflictPredicate) { + valueQuery += 'WHERE ' + this.whereItemsQuery(conflictPredicate, options) + ' '; + } + + valueQuery += + options.onConflict; } valueQuery += ' RETURNING ' + options.returning; } else { @@ -338,21 +345,30 @@ class QueryGenerator { options.updateOnConflict = _.extend({ constraint: '', target: primaryKeyField, - update: allAttributes.filter(value => value !== primaryKeyField) + update: {} }, options.updateOnConflict || {}); + if (!options.updateOnConflict.update.columns) { + options.updateOnConflict.update.columns = allAttributes.filter(value => value !== primaryKeyField); + } const { constraint, target, update } = options.updateOnConflict; let conflict = ''; if (constraint) { conflict = ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint); } else { - conflict = ' ON CONFLICT (' + this.quoteIdentifier(target) + ')'; + const targetArr = Array.isArray(target) ? target : [target]; + const conflictTarget = targetArr.map(e => this.quoteIdentifier(e)).join(', '); + conflict = ' ON CONFLICT (' + conflictTarget + ')'; + + if (update.where) { + conflict += ' WHERE ' + this.whereItemsQuery(update.where, options); + } } if (update === false) { onConflictKeyUpdate += ' DO NOTHING '; } else { - onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + update.map(attr => { + onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + update.columns.map(attr => { const field = rawAttributes[attr] ? rawAttributes[attr].fieldName : attr; const key = this.quoteIdentifier(field); return key + ' = EXCLUDED.' + key; diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index b5f38cd3cec1..17d9f8131652 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -388,12 +388,16 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { options.isPgUpsert = true; if (!options.conflict) { options.conflict = { - target: uniqueKeys, - update: getUpdateArray(updateValues, uniqueKeys) + target: uniqueKeys + }; + } + if (!options.conflict.update) { + options.conflict.update = { + columns: getUpdateArray(updateValues, uniqueKeys) }; } - if (options.conflict.update === false) { + if (options.conflict.update.columns === false) { options.onConflict = ' DO NOTHING'; } else { options.conflict.target = options.conflict.target || uniqueKeys; @@ -401,18 +405,18 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { options.conflict.target = [options.conflict.target]; } options.conflict.target = options.conflict.target.map(key => fieldMap[key] || key); - if (!options.conflict.update) { - options.conflict.update = getUpdateArray(updateValues, options.conflict.target); + if (!options.conflict.update.columns) { + options.conflict.update.columns = getUpdateArray(updateValues, options.conflict.target); } if (rawAttributes.updatedAt) { insertValues.updatedAt = new Date(); - if (options.conflict.update.indexOf('updatedAt') === -1) { - options.conflict.update.push('updatedAt'); + if (options.conflict.update.columns.indexOf('updatedAt') === -1) { + options.conflict.update.columns.push('updatedAt'); } } - options.onConflict = 'DO UPDATE SET ' + options.conflict.update.map(key => { + options.onConflict = 'DO UPDATE SET ' + options.conflict.update.columns.map(key => { key = this.quoteIdentifier(key); return key + ' = EXCLUDED.' + key; }).join(', '); From d7dfb9d55dc958465bf63fc740c0fc3302d53ddc Mon Sep 17 00:00:00 2001 From: Andrew Heuermann Date: Fri, 12 Oct 2018 10:09:19 -0500 Subject: [PATCH 06/10] Allowing where for on conflict update. Renaming conflict predicate to indexPredicase based on postgres docs and using where for update clause --- lib/dialects/abstract/query-generator.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index cb3e589e1ec3..78ab7724a437 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -143,9 +143,9 @@ class QueryGenerator { const conflictTarget = target.map(e => this.quoteIdentifier(e)).join(', '); valueQuery += ' ON CONFLICT (' + conflictTarget + ') '; - const conflictPredicate = options.conflict.update.where; - if (conflictPredicate) { - valueQuery += 'WHERE ' + this.whereItemsQuery(conflictPredicate, options) + ' '; + const indexPredicate = options.conflict.indexPredicate; + if (indexPredicate) { + valueQuery += 'WHERE ' + this.whereItemsQuery(indexPredicate, options) + ' '; } valueQuery += + options.onConflict; @@ -351,7 +351,7 @@ class QueryGenerator { options.updateOnConflict.update.columns = allAttributes.filter(value => value !== primaryKeyField); } - const { constraint, target, update } = options.updateOnConflict; + const { constraint, target, update, indexPredicate } = options.updateOnConflict; let conflict = ''; if (constraint) { conflict = ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint); @@ -360,7 +360,7 @@ class QueryGenerator { const conflictTarget = targetArr.map(e => this.quoteIdentifier(e)).join(', '); conflict = ' ON CONFLICT (' + conflictTarget + ')'; - if (update.where) { + if (indexPredicate) { conflict += ' WHERE ' + this.whereItemsQuery(update.where, options); } } @@ -373,6 +373,11 @@ class QueryGenerator { const key = this.quoteIdentifier(field); return key + ' = EXCLUDED.' + key; }).join(','); + + if (update.where) { + onConflictKeyUpdate += ' WHERE ' + this.whereItemsQuery(update.where, options); + } + } } From df3ab96e24af7ffa220fb52a883bd0578fac1704 Mon Sep 17 00:00:00 2001 From: Andrew Heuermann Date: Fri, 12 Oct 2018 12:12:45 -0500 Subject: [PATCH 07/10] 1. Mapping where fields 2: Include table alias to prevent ambiguity --- lib/dialects/abstract/query-generator.js | 2 +- lib/model.js | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 78ab7724a437..4ea52b3c608c 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -375,7 +375,7 @@ class QueryGenerator { }).join(','); if (update.where) { - onConflictKeyUpdate += ' WHERE ' + this.whereItemsQuery(update.where, options); + onConflictKeyUpdate += ' WHERE ' + this.getWhereConditions(update.where, tableName, options.model, options); } } diff --git a/lib/model.js b/lib/model.js index b659be377415..27ce262edff8 100644 --- a/lib/model.js +++ b/lib/model.js @@ -2515,6 +2515,9 @@ class Model { if (options.updateOnDuplicate && dialect !== 'mysql') { return Promise.reject(new Error(`${dialect} does not support the updateOnDuplicate option.`)); } + if (options.updateOnConflict && dialect !== 'postgres') { + return Promise.reject(new Error(`${dialect} does not support the updateOnConflict option.`)); + } if (options.updateOnDuplicate !== undefined) { if (_.isArray(options.updateOnDuplicate) && options.updateOnDuplicate.length) { @@ -2527,6 +2530,22 @@ class Model { } } + if (options.updateOnConflict !== undefined && options.updateOnConflict.update) { + const where = options.updateOnConflict.update.where; + const columns = options.updateOnConflict.update.columns; + if (columns && _.isArray(columns) && columns.length) { + options.updateOnConflict.update.columns = _.intersection( + _.without(Object.keys(this.tableAttributes), this._timestampAttributes.createdAt), + columns + ); + } else { + return Promise.reject(new Error('updateOnConflict.update.columns option only supports non-empty array.')); + } + if (where) { + options.updateOnConflict.update.where = Utils.mapWhereFieldNames(where, this); + } + } + options.model = this; const createdAtAttr = this._timestampAttributes.createdAt; From 210c6277c1aca5397c863dcff489d0a6da6dc581 Mon Sep 17 00:00:00 2001 From: Andrew Heuermann Date: Fri, 12 Oct 2018 12:44:46 -0500 Subject: [PATCH 08/10] Use field instead of fieldName --- lib/dialects/abstract/query-generator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 4ea52b3c608c..62d193bd6c9b 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -369,7 +369,7 @@ class QueryGenerator { onConflictKeyUpdate += ' DO NOTHING '; } else { onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + update.columns.map(attr => { - const field = rawAttributes[attr] ? rawAttributes[attr].fieldName : attr; + const field = rawAttributes[attr] ? rawAttributes[attr].field : attr; const key = this.quoteIdentifier(field); return key + ' = EXCLUDED.' + key; }).join(','); From f5d0e1a6fc2c5c531710613e9033b42c3ae4bd8f Mon Sep 17 00:00:00 2001 From: Andrew Heuermann Date: Sat, 24 Nov 2018 10:56:40 -0600 Subject: [PATCH 09/10] Fixing on conflict do nothing and composite primary key --- lib/dialects/abstract/query-generator.js | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 62d193bd6c9b..e99dc9fd4505 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -344,31 +344,32 @@ class QueryGenerator { const { primaryKeyField, rawAttributes } = options.model; options.updateOnConflict = _.extend({ constraint: '', - target: primaryKeyField, + target: null, update: {} }, options.updateOnConflict || {}); - if (!options.updateOnConflict.update.columns) { + if (options.updateOnConflict.update !== false && !options.updateOnConflict.update.columns) { options.updateOnConflict.update.columns = allAttributes.filter(value => value !== primaryKeyField); } const { constraint, target, update, indexPredicate } = options.updateOnConflict; - let conflict = ''; if (constraint) { - conflict = ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint); + onConflictKeyUpdate += ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint); } else { - const targetArr = Array.isArray(target) ? target : [target]; - const conflictTarget = targetArr.map(e => this.quoteIdentifier(e)).join(', '); - conflict = ' ON CONFLICT (' + conflictTarget + ')'; - + onConflictKeyUpdate += ' ON CONFLICT '; + if (target) { + const targetArr = Array.isArray(target) ? target : [target]; + const conflictTarget = targetArr.map(e => this.quoteIdentifier(e)).join(', '); + onConflictKeyUpdate += '(' + conflictTarget + ') '; + } if (indexPredicate) { - conflict += ' WHERE ' + this.whereItemsQuery(update.where, options); + onConflictKeyUpdate += ' WHERE ' + this.whereItemsQuery(indexPredicate, options); } } if (update === false) { onConflictKeyUpdate += ' DO NOTHING '; } else { - onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + update.columns.map(attr => { + onConflictKeyUpdate += ' DO UPDATE SET ' + update.columns.map(attr => { const field = rawAttributes[attr] ? rawAttributes[attr].field : attr; const key = this.quoteIdentifier(field); return key + ' = EXCLUDED.' + key; From 85a4b0d42585ad1b3fe16d8876dd1fe09d41cf5f Mon Sep 17 00:00:00 2001 From: Andrew Heuermann Date: Tue, 27 Nov 2018 10:59:39 -0600 Subject: [PATCH 10/10] Add a default target unless on conflict do nothing --- lib/dialects/abstract/query-generator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index e99dc9fd4505..aaf6a6732412 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -344,7 +344,7 @@ class QueryGenerator { const { primaryKeyField, rawAttributes } = options.model; options.updateOnConflict = _.extend({ constraint: '', - target: null, + target: options.updateOnConflict.update === false ? null : primaryKeyField, update: {} }, options.updateOnConflict || {}); if (options.updateOnConflict.update !== false && !options.updateOnConflict.update.columns) {