From 49cc48999e1ad5ae7752ede01f2f431ebf6222ee Mon Sep 17 00:00:00 2001 From: inDream Date: Thu, 21 Jul 2016 21:19:32 +0800 Subject: [PATCH 01/16] Init commit --- lib/dialects/abstract/query-generator.js | 46 ++++++++++++- lib/dialects/postgres/index.js | 1 + lib/dialects/postgres/query-generator.js | 85 +++++++++++++++++++----- lib/dialects/postgres/query.js | 8 ++- test/integration/model/upsert.test.js | 25 ++++--- 5 files changed, 137 insertions(+), 28 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index c3a837dee37b..6bb20a10b2f1 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -131,6 +131,22 @@ class QueryGenerator { emptyQuery += ' VALUES ()'; } + if (this.dialect === 'postgres') { + if (options.onConflict && semver.gte(this.sequelize.options.databaseVersion, '9.5.0')) { + const constraint = options.conflict.constraint; + const target = options.conflict.target; + if (constraint) { + valueQuery += ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint) + options.onConflict; + } else if (target) { + const conflictTarget = target.map(e => this.quoteIdentifier(e)).join(', '); + valueQuery += ' ON CONFLICT (' + conflictTarget + ') ' + options.onConflict; + } + valueQuery += ' RETURNING ' + options.returning; + } else { + options.onConflictFallback = true; + } + } + if (this._dialect.supports.returnValues && options.returning) { if (this._dialect.supports.returnValues.returning) { valueQuery += ' RETURNING *'; @@ -271,11 +287,12 @@ class QueryGenerator { options = options || {}; fieldMappedAttributes = fieldMappedAttributes || {}; - const query = 'INSERT<%= ignoreDuplicates %> INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %><%= onDuplicateKeyUpdate %><%= returning %>;'; + const query = 'INSERT<%= ignoreDuplicates %> INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %><%= onDuplicateKeyUpdate %><%= onConflictKeyUpdate %><%= returning %>;'; const tuples = []; const serials = {}; const allAttributes = []; let onDuplicateKeyUpdate = ''; + let onConflictKeyUpdate = ''; for (const fieldValueHash of fieldValueHashes) { _.forOwn(fieldValueHash, (value, key) => { @@ -313,12 +330,39 @@ class QueryGenerator { }).join(','); } + if (this._dialect.supports.updateOnConflict && options.updateOnConflict) { + options.updateOnConflict = Utils._.extend({ + target: 'id', + update: attrValueHashes.map(value => value.name).filter(value => value !== 'id') + }, options.updateOnConflict || {}); + + let conflict = ''; + const constraint = this.quoteIdentifier(options.updateOnConflict.constraint); + const target = this.quoteIdentifier(options.updateOnConflict.target); + if (constraint) { + conflict = ' ON CONFLICT ON CONSTRAINT ' + target; + } else { + conflict = ' ON CONFLICT (' + target + ')'; + } + + if (options.updateOnConflict.update === false) { + onConflictKeyUpdate += ' DO NOTHING '; + } else { + onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + options.updateOnConflict.update.map(attr => { + const field = rawAttributes && rawAttributes[attr] && rawAttributes[attr].field || attr; + const key = this.quoteIdentifier(field); + return key + ' = EXCLUDED.' + key; + }).join(','); + } + } + const replacements = { ignoreDuplicates: options.ignoreDuplicates ? this._dialect.supports.ignoreDuplicates : '', table: this.quoteTable(tableName), attributes: allAttributes.map(attr => this.quoteIdentifier(attr)).join(','), tuples: tuples.join(','), onDuplicateKeyUpdate, + onConflictKeyUpdate, returning: this._dialect.supports.returnValues && options.returning ? ' RETURNING *' : '' }; diff --git a/lib/dialects/postgres/index.js b/lib/dialects/postgres/index.js index af1560afbffb..dd430677b925 100644 --- a/lib/dialects/postgres/index.js +++ b/lib/dialects/postgres/index.js @@ -50,6 +50,7 @@ PostgresDialect.prototype.supports = _.merge(_.cloneDeep(AbstractDialect.prototy JSONB: true, HSTORE: true, deferrableConstraints: true, + updateOnConflict: true, searchPath: true }); diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index edc0fb45c1bd..34d74bb4dc57 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -349,22 +349,75 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { } upsertQuery(tableName, insertValues, updateValues, where, model, options) { - const primaryField = this.quoteIdentifier(model.primaryKeyField); - - const upsertOptions = _.defaults({ bindParam: false }, options); - const insert = this.insertQuery(tableName, insertValues, model.rawAttributes, upsertOptions); - const update = this.updateQuery(tableName, updateValues, where, upsertOptions, model.rawAttributes); - - insert.query = insert.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); - update.query = update.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); - - return this.exceptionFn( - 'sequelize_upsert', - tableName, - 'OUT created boolean, OUT primary_key text', - `${insert.query} created := true;`, - `${update.query}; created := false` - ); + const rawAttributes = model.rawAttributes; + const indexes = model.options.indexes; + const uniqueKeys = Object.keys(rawAttributes).filter(e => rawAttributes[e].unique); + const uniqueIndexes = indexes.filter(e => e.unique).map(e => e.name); + const uniqueCount = uniqueKeys.length + uniqueIndexes.length; + if (uniqueCount > 1 && (!options.conflict || options.conflict && !options.conflict.constraint) || + semver.lt(this.sequelize.options.databaseVersion, '9.5.0')) { + options.onConflictFallback = true; + const primaryField = this.quoteIdentifier(model.primaryKeyField); + + const upsertOptions = _.defaults({ bindParam: false }, options); + const insert = this.insertQuery(tableName, insertValues, model.rawAttributes, upsertOptions); + const update = this.updateQuery(tableName, updateValues, where, upsertOptions, model.rawAttributes); + + insert.query = insert.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); + update.query = update.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); + + return this.exceptionFn( + 'sequelize_upsert', + tableName, + 'OUT created boolean, OUT primary_key text', + `${insert.query} created := true;`, + `${update.query}; created := false` + ); + } + + if (!options.conflict) { + let target = 'id'; + for (const key of Object.keys(rawAttributes)) { + if (rawAttributes[key].primaryKey) { + target = rawAttributes[key].field; + } + } + options.conflict = { + target, + update: Object.keys(updateValues).filter(value => value !== target) + }; + } + + if (options.conflict.update === false) { + options.onConflict = ' DO NOTHING'; + } else { + options.conflict.target = options.conflict.target || uniqueKeys; + if (!Array.isArray(options.conflict.target)) { + options.conflict.target = [options.conflict.target]; + } + if (!options.conflict.update) { + options.conflict.update = Object.keys(updateValues) + .filter(value => options.conflict.target.indexOf(value) === -1); + } + + if (rawAttributes.updatedAt) { + insertValues.updatedAt = new Date(); + if (options.conflict.update.indexOf('updatedAt') === -1) { + options.conflict.update.push('updatedAt'); + } + } + + options.onConflict = 'DO UPDATE SET ' + options.conflict.update.map(key => { + key = this.quoteIdentifier(key); + return key + ' = EXCLUDED.' + key; + }).join(', '); + } + + if (options.returning === undefined) { + options.returning = '*'; + } + + return this.insertQuery(tableName, insertValues, rawAttributes, options); } truncateTableQuery(tableName, options = {}) { diff --git a/lib/dialects/postgres/query.js b/lib/dialects/postgres/query.js index 44747c380309..42f87e0c082d 100644 --- a/lib/dialects/postgres/query.js +++ b/lib/dialects/postgres/query.js @@ -249,7 +249,13 @@ class Query extends AbstractQuery { } else if (QueryTypes.BULKDELETE === this.options.type) { return parseInt(rowCount, 10); } else if (this.isUpsertQuery()) { - return rows[0]; + if (this.options.onConflictFallback) { + return rows[0]; + } else { + const item = this.options.plain ? rows[0] : + this.options.instance.build(rows[0]); + return this.options.returning === true ? item : rows.length; + } } else if (this.isInsertQuery() || this.isUpdateQuery()) { if (this.instance && this.instance.dataValues) { for (const key in rows[0]) { diff --git a/test/integration/model/upsert.test.js b/test/integration/model/upsert.test.js index 2b32426f37cd..08ad160465a5 100644 --- a/test/integration/model/upsert.test.js +++ b/test/integration/model/upsert.test.js @@ -86,7 +86,8 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); it('works with upsert on a composite key', function() { - return this.User.upsert({ foo: 'baz', bar: 19, username: 'john' }).bind(this).then(function(created) { + const options = { conflict: { constraint: 'users_foo_bar_key' } }; + return this.User.upsert({ foo: 'baz', bar: 19, username: 'john' }, options).bind(this).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; } else { @@ -94,7 +95,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { } this.clock.tick(1000); - return this.User.upsert({ foo: 'baz', bar: 19, username: 'doe' }); + return this.User.upsert({ foo: 'baz', bar: 19, username: 'doe' }, options); }).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -142,12 +143,13 @@ describe(Support.getTestDialectTeaser('Model'), () => { }, username: DataTypes.STRING }); + const options = { conflict: { constraint: 'users_pkey' } }; return User.sync({ force: true }).bind(this).then(() => { return Promise.all([ // Create two users - User.upsert({ a: 'a', b: 'b', username: 'john' }), - User.upsert({ a: 'a', b: 'a', username: 'curt' }) + User.upsert({ a: 'a', b: 'b', username: 'john' }, options), + User.upsert({ a: 'a', b: 'a', username: 'curt' }, options), ]); }).spread(function(created1, created2) { if (dialect === 'sqlite') { @@ -160,7 +162,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { this.clock.tick(1000); // Update the first one - return User.upsert({ a: 'a', b: 'b', username: 'doe' }); + return User.upsert({ a: 'a', b: 'b', username: 'doe' }, options); }).then(created => { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -268,7 +270,8 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); it('works with primary key using .field', function() { - return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'first' }).bind(this).then(function(created) { + const options = { conflict: { target: 'userId' } }; + return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'first' }, options).bind(this).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; } else { @@ -276,7 +279,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { } this.clock.tick(1000); - return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'second' }); + return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'second' }, options); }).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -464,14 +467,15 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); return User.sync({ force: true }).then(() => { - return User.upsert({ name: 'user1', address: 'address', city: 'City' }) + const options = { conflict: { target: ['name', 'address'] } }; + return User.upsert({ name: 'user1', address: 'address', city: 'City' }, options) .then(created => { if (dialect === 'sqlite') { expect(created).to.be.undefined; } else { expect(created).to.be.ok; } - return User.upsert({ name: 'user1', address: 'address', city: 'New City' }); + return User.upsert({ name: 'user1', address: 'address', city: 'New City' }, options); }).then(created => { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -537,7 +541,8 @@ describe(Support.getTestDialectTeaser('Model'), () => { // this record is soft deleted User.create({ name: 'user3', deletedAt: -Infinity }) ]).then(() => { - return User.upsert({ name: 'user1', address: 'address' }); + const options = { conflict: { target: ['name', 'deletedAt'] } }; + return User.upsert({ name: 'user1', address: 'address' }, options); }).then(() => { return User.findAll({ where: { address: null } From f574aa95b83fa1dd26ec886363f2ab2a6ea20406 Mon Sep 17 00:00:00 2001 From: inDream Date: Tue, 10 Oct 2017 14:13:18 +0800 Subject: [PATCH 02/16] Handle postgres version 10 --- lib/dialects/abstract/connection-manager.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/dialects/abstract/connection-manager.js b/lib/dialects/abstract/connection-manager.js index 17c0f3d026b6..15597d7cc67b 100644 --- a/lib/dialects/abstract/connection-manager.js +++ b/lib/dialects/abstract/connection-manager.js @@ -256,6 +256,10 @@ class ConnectionManager { _options.logging.__testLoggingFn = true; return this.sequelize.databaseVersion(_options).then(version => { + if (this.dialectName === 'postgres' && _.isString(version) && + !version.match(/\.\d+\./)) { + version += '.0'; + } this.sequelize.options.databaseVersion = semver.valid(version) ? version : this.defaultVersion; this.versionPromise = null; From b58aa55ea4e1ee28de7b121e424e3b6b82fd7cc6 Mon Sep 17 00:00:00 2001 From: inDream Date: Sun, 28 Jan 2018 23:18:20 +0800 Subject: [PATCH 03/16] Update for new return values --- lib/dialects/abstract/query-generator.js | 2 +- lib/dialects/postgres/query-generator.js | 31 ++++++++++++++---------- lib/dialects/postgres/query.js | 6 ++--- lib/model.js | 3 +++ lib/query-interface.js | 4 +-- 5 files changed, 27 insertions(+), 19 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 6bb20a10b2f1..14548afb936e 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -147,7 +147,7 @@ class QueryGenerator { } } - if (this._dialect.supports.returnValues && options.returning) { + if (this._dialect.supports.returnValues && options.returning && !options.isPgUpsert) { if (this._dialect.supports.returnValues.returning) { valueQuery += ' RETURNING *'; emptyQuery += ' RETURNING *'; diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index 34d74bb4dc57..956d598b073e 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -352,8 +352,17 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { const rawAttributes = model.rawAttributes; const indexes = model.options.indexes; const uniqueKeys = Object.keys(rawAttributes).filter(e => rawAttributes[e].unique); + if (!uniqueKeys.length) { + uniqueKeys.push(model.primaryKeyField); + } const uniqueIndexes = indexes.filter(e => e.unique).map(e => e.name); const uniqueCount = uniqueKeys.length + uniqueIndexes.length; + const fields = Object.keys(model.fieldAttributeMap); + const fieldMap = {}; + fields.forEach(field => { + fieldMap[model.fieldAttributeMap[field]] = field; + }); + if (uniqueCount > 1 && (!options.conflict || options.conflict && !options.conflict.constraint) || semver.lt(this.sequelize.options.databaseVersion, '9.5.0')) { options.onConflictFallback = true; @@ -375,16 +384,14 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { ); } + const getUpdateArray = (values, target) => + Object.keys(values).map(key => fieldMap[key] || key).filter(value => target.indexOf(value) === -1); + + options.isPgUpsert = true; if (!options.conflict) { - let target = 'id'; - for (const key of Object.keys(rawAttributes)) { - if (rawAttributes[key].primaryKey) { - target = rawAttributes[key].field; - } - } options.conflict = { - target, - update: Object.keys(updateValues).filter(value => value !== target) + target: uniqueKeys, + update: getUpdateArray(updateValues, uniqueKeys) }; } @@ -395,9 +402,9 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { if (!Array.isArray(options.conflict.target)) { options.conflict.target = [options.conflict.target]; } + options.conflict.target = options.conflict.target.map(key => fieldMap[key] || key); if (!options.conflict.update) { - options.conflict.update = Object.keys(updateValues) - .filter(value => options.conflict.target.indexOf(value) === -1); + options.conflict.update = getUpdateArray(updateValues, options.conflict.target); } if (rawAttributes.updatedAt) { @@ -413,9 +420,7 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { }).join(', '); } - if (options.returning === undefined) { - options.returning = '*'; - } + options.returning = '*, (xmax = 0)::bool AS xmax'; return this.insertQuery(tableName, insertValues, rawAttributes, options); } diff --git a/lib/dialects/postgres/query.js b/lib/dialects/postgres/query.js index 42f87e0c082d..9692fd8571aa 100644 --- a/lib/dialects/postgres/query.js +++ b/lib/dialects/postgres/query.js @@ -252,9 +252,9 @@ class Query extends AbstractQuery { if (this.options.onConflictFallback) { return rows[0]; } else { - const item = this.options.plain ? rows[0] : - this.options.instance.build(rows[0]); - return this.options.returning === true ? item : rows.length; + this.options.fieldMap = this.options.model.fieldAttributeMap; + const items = this.options.plain ? rows : this.handleSelectQuery(rows); + return [items[0], rows[0].xmax]; } } else if (this.isInsertQuery() || this.isUpdateQuery()) { if (this.instance && this.instance.dataValues) { diff --git a/lib/model.js b/lib/model.js index 88709e5d0dd4..51e77098439c 100644 --- a/lib/model.js +++ b/lib/model.js @@ -2448,6 +2448,9 @@ class Model { }).then(() => { return this.QueryInterface.upsert(this.getTableName(options), insertValues, updateValues, instance.where(), this, options); }).spread((created, primaryKey) => { + if (primaryKey === true || primaryKey === false) { + return options.returning === true ? [created, primaryKey] : primaryKey; + } if (options.returning === true && primaryKey) { return this.findById(primaryKey, options).then(record => [record, created]); } diff --git a/lib/query-interface.js b/lib/query-interface.js index 6dbb9a724ab0..7f04a2f4d2a6 100644 --- a/lib/query-interface.js +++ b/lib/query-interface.js @@ -875,13 +875,13 @@ class QueryInterface { where = { [Op.or]: wheres }; options.type = QueryTypes.UPSERT; - options.raw = true; + options.raw = options.isPgUpsert || false; const sql = this.QueryGenerator.upsertQuery(tableName, insertValues, updateValues, where, model, options); return this.sequelize.query(sql, options).then(result => { switch (this.sequelize.options.dialect) { case 'postgres': - return [result.created, result.primary_key]; + return options.isPgUpsert ? result : [result.created, result.primary_key]; case 'mssql': return [ From 725d40f0bff0759d875ce30f701166efb92959ff Mon Sep 17 00:00:00 2001 From: inDream Date: Thu, 12 Apr 2018 03:15:32 +0800 Subject: [PATCH 04/16] Fix bulkCreate and add test --- lib/dialects/abstract/query-generator.js | 21 +++++++++-------- test/integration/model/upsert.test.js | 29 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 14548afb936e..7926b0d6a6f6 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -331,25 +331,26 @@ class QueryGenerator { } if (this._dialect.supports.updateOnConflict && options.updateOnConflict) { - options.updateOnConflict = Utils._.extend({ - target: 'id', - update: attrValueHashes.map(value => value.name).filter(value => value !== 'id') + const { primaryKeyField, rawAttributes } = options.model; + options.updateOnConflict = _.extend({ + constraint: '', + target: primaryKeyField, + update: allAttributes.filter(value => value !== primaryKeyField) }, options.updateOnConflict || {}); + const { constraint, target, update } = options.updateOnConflict; let conflict = ''; - const constraint = this.quoteIdentifier(options.updateOnConflict.constraint); - const target = this.quoteIdentifier(options.updateOnConflict.target); if (constraint) { - conflict = ' ON CONFLICT ON CONSTRAINT ' + target; + conflict = ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint); } else { - conflict = ' ON CONFLICT (' + target + ')'; + conflict = ' ON CONFLICT (' + this.quoteIdentifier(target) + ')'; } - if (options.updateOnConflict.update === false) { + if (update === false) { onConflictKeyUpdate += ' DO NOTHING '; } else { - onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + options.updateOnConflict.update.map(attr => { - const field = rawAttributes && rawAttributes[attr] && rawAttributes[attr].field || attr; + onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + update.map(attr => { + const field = rawAttributes[attr] ? rawAttributes[attr].fieldName : attr; const key = this.quoteIdentifier(field); return key + ' = EXCLUDED.' + key; }).join(','); diff --git a/test/integration/model/upsert.test.js b/test/integration/model/upsert.test.js index 08ad160465a5..b37fd377d1de 100644 --- a/test/integration/model/upsert.test.js +++ b/test/integration/model/upsert.test.js @@ -552,6 +552,35 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); }); }); + + it('works with bulkCreate', function() { + const User = this.sequelize.define('User', { + name: { + type: DataTypes.STRING, + primaryKey: true + }, + address: DataTypes.STRING + }); + + return User.sync({ force: true }).then(() => { + return User.bulkCreate([ + { name: 'user1', address: 'user1' }, + { name: 'user2' } + ]).then(() => { + return User.bulkCreate([ + { name: 'user1' }, + { name: 'user2', address: 'user2' } + ], { updateOnConflict: true }); + }).then(() => { + return User.findAll({ + where: { address: null } + }); + }).then(users => { + expect(users).to.have.lengthOf(1); + expect(users[0].name).to.equal('user1'); + }); + }); + }); } if (current.dialect.supports.returnValues) { From faecb9139cc2e758bd35efc8c16a5e3274a3c7fc Mon Sep 17 00:00:00 2001 From: Noah Prince Date: Fri, 22 Jun 2018 17:41:59 -0500 Subject: [PATCH 05/16] feat(query-generator): Allow index predicates in postgres upsert. --- lib/dialects/abstract/query-generator.js | 25 +++++++++++++++++++----- lib/dialects/postgres/query-generator.js | 20 +++++++++++-------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 7926b0d6a6f6..674de770a8e8 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -134,12 +134,19 @@ class QueryGenerator { if (this.dialect === 'postgres') { if (options.onConflict && semver.gte(this.sequelize.options.databaseVersion, '9.5.0')) { const constraint = options.conflict.constraint; - const target = options.conflict.target; + const target = Array.isArray(options.conflict.target) ? options.conflict.target : [options.conflict.target]; if (constraint) { valueQuery += ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint) + options.onConflict; } else if (target) { const conflictTarget = target.map(e => this.quoteIdentifier(e)).join(', '); - valueQuery += ' ON CONFLICT (' + conflictTarget + ') ' + options.onConflict; + valueQuery += ' ON CONFLICT (' + conflictTarget + ') '; + + const conflictPredicate = options.conflict.update.where; + if (conflictPredicate) { + valueQuery += 'WHERE ' + this.whereItemsQuery(conflictPredicate, options) + ' '; + } + + valueQuery += + options.onConflict; } valueQuery += ' RETURNING ' + options.returning; } else { @@ -335,7 +342,9 @@ class QueryGenerator { options.updateOnConflict = _.extend({ constraint: '', target: primaryKeyField, - update: allAttributes.filter(value => value !== primaryKeyField) + update: _.extend({ + columns: allAttributes.filter(value => value !== primaryKeyField) + }) }, options.updateOnConflict || {}); const { constraint, target, update } = options.updateOnConflict; @@ -343,13 +352,19 @@ class QueryGenerator { if (constraint) { conflict = ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint); } else { - conflict = ' ON CONFLICT (' + this.quoteIdentifier(target) + ')'; + const targetArr = Array.isArray(target) ? target : [target]; + const conflictTarget = targetArr.map(e => this.quoteIdentifier(e)).join(', '); + conflict = ' ON CONFLICT (' + conflictTarget + ')'; + + if (update.where) { + conflict += ' WHERE ' + this.whereItemsQuery(update.where, options); + } } if (update === false) { onConflictKeyUpdate += ' DO NOTHING '; } else { - onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + update.map(attr => { + onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + update.columns.map(attr => { const field = rawAttributes[attr] ? rawAttributes[attr].fieldName : attr; const key = this.quoteIdentifier(field); return key + ' = EXCLUDED.' + key; diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index 956d598b073e..8a33bb4c8147 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -390,12 +390,16 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { options.isPgUpsert = true; if (!options.conflict) { options.conflict = { - target: uniqueKeys, - update: getUpdateArray(updateValues, uniqueKeys) + target: uniqueKeys }; } + if (!options.conflict.update) { + options.conflict.update = { + columns: getUpdateArray(updateValues, uniqueKeys) + } + } - if (options.conflict.update === false) { + if (options.conflict.update.columns === false) { options.onConflict = ' DO NOTHING'; } else { options.conflict.target = options.conflict.target || uniqueKeys; @@ -403,18 +407,18 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { options.conflict.target = [options.conflict.target]; } options.conflict.target = options.conflict.target.map(key => fieldMap[key] || key); - if (!options.conflict.update) { - options.conflict.update = getUpdateArray(updateValues, options.conflict.target); + if (!options.conflict.update.columns) { + options.conflict.update.columns = getUpdateArray(updateValues, options.conflict.target); } if (rawAttributes.updatedAt) { insertValues.updatedAt = new Date(); - if (options.conflict.update.indexOf('updatedAt') === -1) { - options.conflict.update.push('updatedAt'); + if (options.conflict.update.columns.indexOf('updatedAt') === -1) { + options.conflict.update.columns.push('updatedAt'); } } - options.onConflict = 'DO UPDATE SET ' + options.conflict.update.map(key => { + options.onConflict = 'DO UPDATE SET ' + options.conflict.update.columns.map(key => { key = this.quoteIdentifier(key); return key + ' = EXCLUDED.' + key; }).join(', '); From 5676a3e62c28066fce2516a0d0719d6f79135931 Mon Sep 17 00:00:00 2001 From: Noah Prince Date: Mon, 25 Jun 2018 11:06:07 -0500 Subject: [PATCH 06/16] fix(query-generator): Default update columns --- lib/dialects/abstract/query-generator.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 674de770a8e8..1e98c9a294de 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -342,10 +342,11 @@ class QueryGenerator { options.updateOnConflict = _.extend({ constraint: '', target: primaryKeyField, - update: _.extend({ - columns: allAttributes.filter(value => value !== primaryKeyField) - }) + update: {} }, options.updateOnConflict || {}); + if (!options.updateOnConflict.update.columns) { + options.updateOnConflict.update.columns = allAttributes.filter(value => value !== primaryKeyField); + } const { constraint, target, update } = options.updateOnConflict; let conflict = ''; From a5c72289707d495f59662dcd6c5ed0fe17d574dc Mon Sep 17 00:00:00 2001 From: Bostrong Date: Sun, 22 Jul 2018 19:37:11 +1000 Subject: [PATCH 07/16] debug statements --- lib/dialects/postgres/query-generator.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index 8a33bb4c8147..2e8c02de94f0 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -402,15 +402,20 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { if (options.conflict.update.columns === false) { options.onConflict = ' DO NOTHING'; } else { + options.conflict.target = options.conflict.target || uniqueKeys; + if (!Array.isArray(options.conflict.target)) { options.conflict.target = [options.conflict.target]; } + options.conflict.target = options.conflict.target.map(key => fieldMap[key] || key); if (!options.conflict.update.columns) { options.conflict.update.columns = getUpdateArray(updateValues, options.conflict.target); } + console.log('pgUpsert: 1.) options.conflict.update.columns = ' + JSON.stringify(options.conflict.update.columns)) + if (rawAttributes.updatedAt) { insertValues.updatedAt = new Date(); if (options.conflict.update.columns.indexOf('updatedAt') === -1) { @@ -418,12 +423,16 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { } } + console.log('pgUpsert: 2.) options.conflict.update.columns = ' + JSON.stringify(options.conflict.update.columns)) + options.onConflict = 'DO UPDATE SET ' + options.conflict.update.columns.map(key => { key = this.quoteIdentifier(key); return key + ' = EXCLUDED.' + key; }).join(', '); } + console.log('pgUpsert: 3.) options.onConflict = ' + JSON.stringify(options.onConflict)) + options.returning = '*, (xmax = 0)::bool AS xmax'; return this.insertQuery(tableName, insertValues, rawAttributes, options); From 480cb21a2402943918937de0b60cd8dd0d71c17b Mon Sep 17 00:00:00 2001 From: Bostrong Date: Sun, 22 Jul 2018 20:34:37 +1000 Subject: [PATCH 08/16] debug statements --- lib/dialects/abstract/query-generator.js | 14 ++++++++++++++ lib/dialects/postgres/query-generator.js | 8 +++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 1e98c9a294de..7ae423be568a 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -154,6 +154,8 @@ class QueryGenerator { } } + console.log('abstract_insertQuery_QG: 1.) valueQuery = ' + valueQuery); + if (this._dialect.supports.returnValues && options.returning && !options.isPgUpsert) { if (this._dialect.supports.returnValues.returning) { valueQuery += ' RETURNING *'; @@ -219,6 +221,8 @@ class QueryGenerator { } } + console.log('abstract_insertQuery_QG: 2.) valueQuery = ' + valueQuery); + if (this._dialect.supports['ON DUPLICATE KEY'] && options.onDuplicate) { valueQuery += ' ON DUPLICATE KEY ' + options.onDuplicate; emptyQuery += ' ON DUPLICATE KEY ' + options.onDuplicate; @@ -271,12 +275,22 @@ class QueryGenerator { ].join(' '); } + console.log('abstract_insertQuery_QG: 3.) query = ' + query); + query = _.template(query, this._templateSettings)(replacements); + + + console.log('abstract_insertQuery_QG: 4.) query = ' + query); + // Used by Postgres upsertQuery and calls to here with options.exception set to true const result = { query }; if (options.bindParam !== false) { result.bind = bind; } + + + console.log('abstract_insertQuery_QG: 5.) result = ' + result); + return result; } diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index 2e8c02de94f0..e70bf6dea3b6 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -414,7 +414,7 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { options.conflict.update.columns = getUpdateArray(updateValues, options.conflict.target); } - console.log('pgUpsert: 1.) options.conflict.update.columns = ' + JSON.stringify(options.conflict.update.columns)) + console.log('pgUpsert_QG : 1.) options.conflict.update.columns = ' + JSON.stringify(options.conflict.update.columns)) if (rawAttributes.updatedAt) { insertValues.updatedAt = new Date(); @@ -423,7 +423,7 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { } } - console.log('pgUpsert: 2.) options.conflict.update.columns = ' + JSON.stringify(options.conflict.update.columns)) + console.log('pgUpsert_QG: 2.) options.conflict.update.columns = ' + JSON.stringify(options.conflict.update.columns)) options.onConflict = 'DO UPDATE SET ' + options.conflict.update.columns.map(key => { key = this.quoteIdentifier(key); @@ -431,10 +431,12 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { }).join(', '); } - console.log('pgUpsert: 3.) options.onConflict = ' + JSON.stringify(options.onConflict)) + console.log('pgUpsert_QG: 3.) options.onConflict = ' + JSON.stringify(options.onConflict)) options.returning = '*, (xmax = 0)::bool AS xmax'; + console.log('pgUpsert_QG: 4.) options = ' + JSON.stringify(options)); + return this.insertQuery(tableName, insertValues, rawAttributes, options); } From 92eceeecae5917812cb7463494afa5d4991ebadd Mon Sep 17 00:00:00 2001 From: Bostrong Date: Sun, 22 Jul 2018 20:42:03 +1000 Subject: [PATCH 09/16] debug statements --- lib/dialects/postgres/query-generator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index e70bf6dea3b6..143f92c51c6d 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -435,7 +435,7 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { options.returning = '*, (xmax = 0)::bool AS xmax'; - console.log('pgUpsert_QG: 4.) options = ' + JSON.stringify(options)); + // console.log('pgUpsert_QG: 4.) options = ' + JSON.stringify(options)); return this.insertQuery(tableName, insertValues, rawAttributes, options); } From 378ce0bf08aadcba84335820164ebd7a11881932 Mon Sep 17 00:00:00 2001 From: Bostrong Date: Sun, 22 Jul 2018 20:57:00 +1000 Subject: [PATCH 10/16] fix errornous placement of '+' in abstract_queryGenerator_insertQuery function. --- lib/dialects/abstract/query-generator.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 7ae423be568a..ac7a73b85468 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -146,7 +146,8 @@ class QueryGenerator { valueQuery += 'WHERE ' + this.whereItemsQuery(conflictPredicate, options) + ' '; } - valueQuery += + options.onConflict; + valueQuery += options.onConflict; + } valueQuery += ' RETURNING ' + options.returning; } else { From ab50d0807ee2c4b0148f72f1ae941fd31b347107 Mon Sep 17 00:00:00 2001 From: Bostrong Date: Sun, 22 Jul 2018 22:13:26 +1000 Subject: [PATCH 11/16] Remove debug statements --- lib/dialects/abstract/query-generator.js | 14 +------------- lib/dialects/postgres/query-generator.js | 8 -------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index ac7a73b85468..3003f9b4a6d5 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -147,7 +147,7 @@ class QueryGenerator { } valueQuery += options.onConflict; - + } valueQuery += ' RETURNING ' + options.returning; } else { @@ -155,8 +155,6 @@ class QueryGenerator { } } - console.log('abstract_insertQuery_QG: 1.) valueQuery = ' + valueQuery); - if (this._dialect.supports.returnValues && options.returning && !options.isPgUpsert) { if (this._dialect.supports.returnValues.returning) { valueQuery += ' RETURNING *'; @@ -222,8 +220,6 @@ class QueryGenerator { } } - console.log('abstract_insertQuery_QG: 2.) valueQuery = ' + valueQuery); - if (this._dialect.supports['ON DUPLICATE KEY'] && options.onDuplicate) { valueQuery += ' ON DUPLICATE KEY ' + options.onDuplicate; emptyQuery += ' ON DUPLICATE KEY ' + options.onDuplicate; @@ -276,22 +272,14 @@ class QueryGenerator { ].join(' '); } - console.log('abstract_insertQuery_QG: 3.) query = ' + query); - query = _.template(query, this._templateSettings)(replacements); - - console.log('abstract_insertQuery_QG: 4.) query = ' + query); - // Used by Postgres upsertQuery and calls to here with options.exception set to true const result = { query }; if (options.bindParam !== false) { result.bind = bind; } - - console.log('abstract_insertQuery_QG: 5.) result = ' + result); - return result; } diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index 143f92c51c6d..a19d1025dcaf 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -414,8 +414,6 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { options.conflict.update.columns = getUpdateArray(updateValues, options.conflict.target); } - console.log('pgUpsert_QG : 1.) options.conflict.update.columns = ' + JSON.stringify(options.conflict.update.columns)) - if (rawAttributes.updatedAt) { insertValues.updatedAt = new Date(); if (options.conflict.update.columns.indexOf('updatedAt') === -1) { @@ -423,20 +421,14 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { } } - console.log('pgUpsert_QG: 2.) options.conflict.update.columns = ' + JSON.stringify(options.conflict.update.columns)) - options.onConflict = 'DO UPDATE SET ' + options.conflict.update.columns.map(key => { key = this.quoteIdentifier(key); return key + ' = EXCLUDED.' + key; }).join(', '); } - console.log('pgUpsert_QG: 3.) options.onConflict = ' + JSON.stringify(options.onConflict)) - options.returning = '*, (xmax = 0)::bool AS xmax'; - // console.log('pgUpsert_QG: 4.) options = ' + JSON.stringify(options)); - return this.insertQuery(tableName, insertValues, rawAttributes, options); } From 64adc9851b20faa381550747c8e6d863ef6ba406 Mon Sep 17 00:00:00 2001 From: inDream Date: Thu, 21 Jul 2016 21:19:32 +0800 Subject: [PATCH 12/16] Init commit --- lib/dialects/abstract/query-generator.js | 46 ++++++++++++- lib/dialects/postgres/index.js | 1 + lib/dialects/postgres/query-generator.js | 85 +++++++++++++++++++----- lib/dialects/postgres/query.js | 8 ++- test/integration/model/upsert.test.js | 25 ++++--- 5 files changed, 137 insertions(+), 28 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 087d4fe95926..eb1af9beb5bc 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -133,6 +133,22 @@ class QueryGenerator { emptyQuery += ' VALUES ()'; } + if (this.dialect === 'postgres') { + if (options.onConflict && semver.gte(this.sequelize.options.databaseVersion, '9.5.0')) { + const constraint = options.conflict.constraint; + const target = options.conflict.target; + if (constraint) { + valueQuery += ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint) + options.onConflict; + } else if (target) { + const conflictTarget = target.map(e => this.quoteIdentifier(e)).join(', '); + valueQuery += ' ON CONFLICT (' + conflictTarget + ') ' + options.onConflict; + } + valueQuery += ' RETURNING ' + options.returning; + } else { + options.onConflictFallback = true; + } + } + if (this._dialect.supports.returnValues && options.returning) { if (this._dialect.supports.returnValues.returning) { valueQuery += ' RETURNING *'; @@ -274,11 +290,12 @@ class QueryGenerator { options = options || {}; fieldMappedAttributes = fieldMappedAttributes || {}; - const query = 'INSERT<%= ignoreDuplicates %> INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %><%= onDuplicateKeyUpdate %><%= onConflictDoNothing %><%= returning %>;'; + const query = 'INSERT<%= ignoreDuplicates %> INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %><%= onDuplicateKeyUpdate %><%= onConflictDoNothing %><%= onConflictKeyUpdate %><%= returning %>;'; const tuples = []; const serials = {}; const allAttributes = []; let onDuplicateKeyUpdate = ''; + let onConflictKeyUpdate = ''; for (const fieldValueHash of fieldValueHashes) { _.forOwn(fieldValueHash, (value, key) => { @@ -316,12 +333,39 @@ class QueryGenerator { }).join(','); } + if (this._dialect.supports.updateOnConflict && options.updateOnConflict) { + options.updateOnConflict = Utils._.extend({ + target: 'id', + update: attrValueHashes.map(value => value.name).filter(value => value !== 'id') + }, options.updateOnConflict || {}); + + let conflict = ''; + const constraint = this.quoteIdentifier(options.updateOnConflict.constraint); + const target = this.quoteIdentifier(options.updateOnConflict.target); + if (constraint) { + conflict = ' ON CONFLICT ON CONSTRAINT ' + target; + } else { + conflict = ' ON CONFLICT (' + target + ')'; + } + + if (options.updateOnConflict.update === false) { + onConflictKeyUpdate += ' DO NOTHING '; + } else { + onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + options.updateOnConflict.update.map(attr => { + const field = rawAttributes && rawAttributes[attr] && rawAttributes[attr].field || attr; + const key = this.quoteIdentifier(field); + return key + ' = EXCLUDED.' + key; + }).join(','); + } + } + const replacements = { ignoreDuplicates: options.ignoreDuplicates ? this._dialect.supports.inserts.ignoreDuplicates : '', table: this.quoteTable(tableName), attributes: allAttributes.map(attr => this.quoteIdentifier(attr)).join(','), tuples: tuples.join(','), onDuplicateKeyUpdate, + onConflictKeyUpdate, returning: this._dialect.supports.returnValues && options.returning ? ' RETURNING *' : '', onConflictDoNothing: options.ignoreDuplicates ? this._dialect.supports.inserts.onConflictDoNothing : '' }; diff --git a/lib/dialects/postgres/index.js b/lib/dialects/postgres/index.js index e1b65df57aac..48e19d303da4 100644 --- a/lib/dialects/postgres/index.js +++ b/lib/dialects/postgres/index.js @@ -53,6 +53,7 @@ PostgresDialect.prototype.supports = _.merge(_.cloneDeep(AbstractDialect.prototy JSONB: true, HSTORE: true, deferrableConstraints: true, + updateOnConflict: true, searchPath: true }); diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index e1f1d239e975..47c28929e048 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -347,22 +347,75 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { } upsertQuery(tableName, insertValues, updateValues, where, model, options) { - const primaryField = this.quoteIdentifier(model.primaryKeyField); - - const upsertOptions = _.defaults({ bindParam: false }, options); - const insert = this.insertQuery(tableName, insertValues, model.rawAttributes, upsertOptions); - const update = this.updateQuery(tableName, updateValues, where, upsertOptions, model.rawAttributes); - - insert.query = insert.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); - update.query = update.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); - - return this.exceptionFn( - 'sequelize_upsert', - tableName, - 'OUT created boolean, OUT primary_key text', - `${insert.query} created := true;`, - `${update.query}; created := false` - ); + const rawAttributes = model.rawAttributes; + const indexes = model.options.indexes; + const uniqueKeys = Object.keys(rawAttributes).filter(e => rawAttributes[e].unique); + const uniqueIndexes = indexes.filter(e => e.unique).map(e => e.name); + const uniqueCount = uniqueKeys.length + uniqueIndexes.length; + if (uniqueCount > 1 && (!options.conflict || options.conflict && !options.conflict.constraint) || + semver.lt(this.sequelize.options.databaseVersion, '9.5.0')) { + options.onConflictFallback = true; + const primaryField = this.quoteIdentifier(model.primaryKeyField); + + const upsertOptions = _.defaults({ bindParam: false }, options); + const insert = this.insertQuery(tableName, insertValues, model.rawAttributes, upsertOptions); + const update = this.updateQuery(tableName, updateValues, where, upsertOptions, model.rawAttributes); + + insert.query = insert.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); + update.query = update.query.replace('RETURNING *', `RETURNING ${primaryField} INTO primary_key`); + + return this.exceptionFn( + 'sequelize_upsert', + tableName, + 'OUT created boolean, OUT primary_key text', + `${insert.query} created := true;`, + `${update.query}; created := false` + ); + } + + if (!options.conflict) { + let target = 'id'; + for (const key of Object.keys(rawAttributes)) { + if (rawAttributes[key].primaryKey) { + target = rawAttributes[key].field; + } + } + options.conflict = { + target, + update: Object.keys(updateValues).filter(value => value !== target) + }; + } + + if (options.conflict.update === false) { + options.onConflict = ' DO NOTHING'; + } else { + options.conflict.target = options.conflict.target || uniqueKeys; + if (!Array.isArray(options.conflict.target)) { + options.conflict.target = [options.conflict.target]; + } + if (!options.conflict.update) { + options.conflict.update = Object.keys(updateValues) + .filter(value => options.conflict.target.indexOf(value) === -1); + } + + if (rawAttributes.updatedAt) { + insertValues.updatedAt = new Date(); + if (options.conflict.update.indexOf('updatedAt') === -1) { + options.conflict.update.push('updatedAt'); + } + } + + options.onConflict = 'DO UPDATE SET ' + options.conflict.update.map(key => { + key = this.quoteIdentifier(key); + return key + ' = EXCLUDED.' + key; + }).join(', '); + } + + if (options.returning === undefined) { + options.returning = '*'; + } + + return this.insertQuery(tableName, insertValues, rawAttributes, options); } truncateTableQuery(tableName, options = {}) { diff --git a/lib/dialects/postgres/query.js b/lib/dialects/postgres/query.js index 03117e96900a..b5c753115e1e 100644 --- a/lib/dialects/postgres/query.js +++ b/lib/dialects/postgres/query.js @@ -249,7 +249,13 @@ class Query extends AbstractQuery { } else if (QueryTypes.BULKDELETE === this.options.type) { return parseInt(rowCount, 10); } else if (this.isUpsertQuery()) { - return rows[0]; + if (this.options.onConflictFallback) { + return rows[0]; + } else { + const item = this.options.plain ? rows[0] : + this.options.instance.build(rows[0]); + return this.options.returning === true ? item : rows.length; + } } else if (this.isInsertQuery() || this.isUpdateQuery()) { if (this.instance && this.instance.dataValues) { for (const key in rows[0]) { diff --git a/test/integration/model/upsert.test.js b/test/integration/model/upsert.test.js index bcb4ed369afb..b4b973b5fcec 100644 --- a/test/integration/model/upsert.test.js +++ b/test/integration/model/upsert.test.js @@ -86,7 +86,8 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); it('works with upsert on a composite key', function() { - return this.User.upsert({ foo: 'baz', bar: 19, username: 'john' }).bind(this).then(function(created) { + const options = { conflict: { constraint: 'users_foo_bar_key' } }; + return this.User.upsert({ foo: 'baz', bar: 19, username: 'john' }, options).bind(this).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; } else { @@ -94,7 +95,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { } this.clock.tick(1000); - return this.User.upsert({ foo: 'baz', bar: 19, username: 'doe' }); + return this.User.upsert({ foo: 'baz', bar: 19, username: 'doe' }, options); }).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -142,12 +143,13 @@ describe(Support.getTestDialectTeaser('Model'), () => { }, username: DataTypes.STRING }); + const options = { conflict: { constraint: 'users_pkey' } }; return User.sync({ force: true }).bind(this).then(() => { return Promise.all([ // Create two users - User.upsert({ a: 'a', b: 'b', username: 'john' }), - User.upsert({ a: 'a', b: 'a', username: 'curt' }) + User.upsert({ a: 'a', b: 'b', username: 'john' }, options), + User.upsert({ a: 'a', b: 'a', username: 'curt' }, options), ]); }).spread(function(created1, created2) { if (dialect === 'sqlite') { @@ -160,7 +162,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { this.clock.tick(1000); // Update the first one - return User.upsert({ a: 'a', b: 'b', username: 'doe' }); + return User.upsert({ a: 'a', b: 'b', username: 'doe' }, options); }).then(created => { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -268,7 +270,8 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); it('works with primary key using .field', function() { - return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'first' }).bind(this).then(function(created) { + const options = { conflict: { target: 'userId' } }; + return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'first' }, options).bind(this).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; } else { @@ -276,7 +279,7 @@ describe(Support.getTestDialectTeaser('Model'), () => { } this.clock.tick(1000); - return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'second' }); + return this.ModelWithFieldPK.upsert({ userId: 42, foo: 'second' }, options); }).then(function(created) { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -464,14 +467,15 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); return User.sync({ force: true }).then(() => { - return User.upsert({ name: 'user1', address: 'address', city: 'City' }) + const options = { conflict: { target: ['name', 'address'] } }; + return User.upsert({ name: 'user1', address: 'address', city: 'City' }, options) .then(created => { if (dialect === 'sqlite') { expect(created).to.be.undefined; } else { expect(created).to.be.ok; } - return User.upsert({ name: 'user1', address: 'address', city: 'New City' }); + return User.upsert({ name: 'user1', address: 'address', city: 'New City' }, options); }).then(created => { if (dialect === 'sqlite') { expect(created).to.be.undefined; @@ -537,7 +541,8 @@ describe(Support.getTestDialectTeaser('Model'), () => { // this record is soft deleted User.create({ name: 'user3', deletedAt: -Infinity }) ]).then(() => { - return User.upsert({ name: 'user1', address: 'address' }); + const options = { conflict: { target: ['name', 'deletedAt'] } }; + return User.upsert({ name: 'user1', address: 'address' }, options); }).then(() => { return User.findAll({ where: { address: null } From 3dfdd7c3da6a5f978ea31f02785cf7257ea3afa5 Mon Sep 17 00:00:00 2001 From: inDream Date: Tue, 10 Oct 2017 14:13:18 +0800 Subject: [PATCH 13/16] Handle postgres version 10 --- lib/dialects/abstract/connection-manager.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/dialects/abstract/connection-manager.js b/lib/dialects/abstract/connection-manager.js index 45040691aa9b..1aba627e3c91 100644 --- a/lib/dialects/abstract/connection-manager.js +++ b/lib/dialects/abstract/connection-manager.js @@ -255,6 +255,10 @@ class ConnectionManager { _options.logging.__testLoggingFn = true; return this.sequelize.databaseVersion(_options).then(version => { + if (this.dialectName === 'postgres' && _.isString(version) && + !version.match(/\.\d+\./)) { + version += '.0'; + } this.sequelize.options.databaseVersion = semver.valid(version) ? version : this.defaultVersion; this.versionPromise = null; From fba14a72f0d8d68c336b00aed1f37cc7f1052298 Mon Sep 17 00:00:00 2001 From: inDream Date: Sun, 28 Jan 2018 23:18:20 +0800 Subject: [PATCH 14/16] Update for new return values --- lib/dialects/abstract/query-generator.js | 2 +- lib/dialects/postgres/query-generator.js | 31 ++++++++++++++---------- lib/dialects/postgres/query.js | 6 ++--- lib/model.js | 3 +++ lib/query-interface.js | 4 +-- 5 files changed, 27 insertions(+), 19 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index eb1af9beb5bc..bf5267c90ca7 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -149,7 +149,7 @@ class QueryGenerator { } } - if (this._dialect.supports.returnValues && options.returning) { + if (this._dialect.supports.returnValues && options.returning && !options.isPgUpsert) { if (this._dialect.supports.returnValues.returning) { valueQuery += ' RETURNING *'; emptyQuery += ' RETURNING *'; diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index 47c28929e048..b5f38cd3cec1 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -350,8 +350,17 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { const rawAttributes = model.rawAttributes; const indexes = model.options.indexes; const uniqueKeys = Object.keys(rawAttributes).filter(e => rawAttributes[e].unique); + if (!uniqueKeys.length) { + uniqueKeys.push(model.primaryKeyField); + } const uniqueIndexes = indexes.filter(e => e.unique).map(e => e.name); const uniqueCount = uniqueKeys.length + uniqueIndexes.length; + const fields = Object.keys(model.fieldAttributeMap); + const fieldMap = {}; + fields.forEach(field => { + fieldMap[model.fieldAttributeMap[field]] = field; + }); + if (uniqueCount > 1 && (!options.conflict || options.conflict && !options.conflict.constraint) || semver.lt(this.sequelize.options.databaseVersion, '9.5.0')) { options.onConflictFallback = true; @@ -373,16 +382,14 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { ); } + const getUpdateArray = (values, target) => + Object.keys(values).map(key => fieldMap[key] || key).filter(value => target.indexOf(value) === -1); + + options.isPgUpsert = true; if (!options.conflict) { - let target = 'id'; - for (const key of Object.keys(rawAttributes)) { - if (rawAttributes[key].primaryKey) { - target = rawAttributes[key].field; - } - } options.conflict = { - target, - update: Object.keys(updateValues).filter(value => value !== target) + target: uniqueKeys, + update: getUpdateArray(updateValues, uniqueKeys) }; } @@ -393,9 +400,9 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { if (!Array.isArray(options.conflict.target)) { options.conflict.target = [options.conflict.target]; } + options.conflict.target = options.conflict.target.map(key => fieldMap[key] || key); if (!options.conflict.update) { - options.conflict.update = Object.keys(updateValues) - .filter(value => options.conflict.target.indexOf(value) === -1); + options.conflict.update = getUpdateArray(updateValues, options.conflict.target); } if (rawAttributes.updatedAt) { @@ -411,9 +418,7 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { }).join(', '); } - if (options.returning === undefined) { - options.returning = '*'; - } + options.returning = '*, (xmax = 0)::bool AS xmax'; return this.insertQuery(tableName, insertValues, rawAttributes, options); } diff --git a/lib/dialects/postgres/query.js b/lib/dialects/postgres/query.js index b5c753115e1e..abc452b747be 100644 --- a/lib/dialects/postgres/query.js +++ b/lib/dialects/postgres/query.js @@ -252,9 +252,9 @@ class Query extends AbstractQuery { if (this.options.onConflictFallback) { return rows[0]; } else { - const item = this.options.plain ? rows[0] : - this.options.instance.build(rows[0]); - return this.options.returning === true ? item : rows.length; + this.options.fieldMap = this.options.model.fieldAttributeMap; + const items = this.options.plain ? rows : this.handleSelectQuery(rows); + return [items[0], rows[0].xmax]; } } else if (this.isInsertQuery() || this.isUpdateQuery()) { if (this.instance && this.instance.dataValues) { diff --git a/lib/model.js b/lib/model.js index 756db5d5084e..b659be377415 100644 --- a/lib/model.js +++ b/lib/model.js @@ -2452,6 +2452,9 @@ class Model { }).then(() => { return this.QueryInterface.upsert(this.getTableName(options), insertValues, updateValues, instance.where(), this, options); }).spread((created, primaryKey) => { + if (primaryKey === true || primaryKey === false) { + return options.returning === true ? [created, primaryKey] : primaryKey; + } if (options.returning === true && primaryKey) { return this.findById(primaryKey, options).then(record => [record, created]); } diff --git a/lib/query-interface.js b/lib/query-interface.js index 639b7ee945ad..f0aaa393620c 100644 --- a/lib/query-interface.js +++ b/lib/query-interface.js @@ -895,13 +895,13 @@ class QueryInterface { where = { [Op.or]: wheres }; options.type = QueryTypes.UPSERT; - options.raw = true; + options.raw = options.isPgUpsert || false; const sql = this.QueryGenerator.upsertQuery(tableName, insertValues, updateValues, where, model, options); return this.sequelize.query(sql, options).then(result => { switch (this.sequelize.options.dialect) { case 'postgres': - return [result.created, result.primary_key]; + return options.isPgUpsert ? result : [result.created, result.primary_key]; case 'mssql': return [ From 2bf949ac18987a690f6d31a0cf2a0b6fb062b4e9 Mon Sep 17 00:00:00 2001 From: inDream Date: Thu, 12 Apr 2018 03:15:32 +0800 Subject: [PATCH 15/16] Fix bulkCreate and add test --- lib/dialects/abstract/query-generator.js | 21 +++++++++-------- test/integration/model/upsert.test.js | 29 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index bf5267c90ca7..12f5e22c0071 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -334,25 +334,26 @@ class QueryGenerator { } if (this._dialect.supports.updateOnConflict && options.updateOnConflict) { - options.updateOnConflict = Utils._.extend({ - target: 'id', - update: attrValueHashes.map(value => value.name).filter(value => value !== 'id') + const { primaryKeyField, rawAttributes } = options.model; + options.updateOnConflict = _.extend({ + constraint: '', + target: primaryKeyField, + update: allAttributes.filter(value => value !== primaryKeyField) }, options.updateOnConflict || {}); + const { constraint, target, update } = options.updateOnConflict; let conflict = ''; - const constraint = this.quoteIdentifier(options.updateOnConflict.constraint); - const target = this.quoteIdentifier(options.updateOnConflict.target); if (constraint) { - conflict = ' ON CONFLICT ON CONSTRAINT ' + target; + conflict = ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint); } else { - conflict = ' ON CONFLICT (' + target + ')'; + conflict = ' ON CONFLICT (' + this.quoteIdentifier(target) + ')'; } - if (options.updateOnConflict.update === false) { + if (update === false) { onConflictKeyUpdate += ' DO NOTHING '; } else { - onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + options.updateOnConflict.update.map(attr => { - const field = rawAttributes && rawAttributes[attr] && rawAttributes[attr].field || attr; + onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + update.map(attr => { + const field = rawAttributes[attr] ? rawAttributes[attr].fieldName : attr; const key = this.quoteIdentifier(field); return key + ' = EXCLUDED.' + key; }).join(','); diff --git a/test/integration/model/upsert.test.js b/test/integration/model/upsert.test.js index b4b973b5fcec..25293b42785b 100644 --- a/test/integration/model/upsert.test.js +++ b/test/integration/model/upsert.test.js @@ -552,6 +552,35 @@ describe(Support.getTestDialectTeaser('Model'), () => { }); }); }); + + it('works with bulkCreate', function() { + const User = this.sequelize.define('User', { + name: { + type: DataTypes.STRING, + primaryKey: true + }, + address: DataTypes.STRING + }); + + return User.sync({ force: true }).then(() => { + return User.bulkCreate([ + { name: 'user1', address: 'user1' }, + { name: 'user2' } + ]).then(() => { + return User.bulkCreate([ + { name: 'user1' }, + { name: 'user2', address: 'user2' } + ], { updateOnConflict: true }); + }).then(() => { + return User.findAll({ + where: { address: null } + }); + }).then(users => { + expect(users).to.have.lengthOf(1); + expect(users[0].name).to.equal('user1'); + }); + }); + }); } if (current.dialect.supports.returnValues) { From 3735c5c23dee96000eeecaf94949aee7aaeeb0d9 Mon Sep 17 00:00:00 2001 From: Noah Prince Date: Fri, 22 Jun 2018 17:41:59 -0500 Subject: [PATCH 16/16] feat(query-generator): Allow index predicates in postgres upsert. --- lib/dialects/abstract/query-generator.js | 26 +++++++++++++++++++----- lib/dialects/postgres/query-generator.js | 20 ++++++++++-------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/lib/dialects/abstract/query-generator.js b/lib/dialects/abstract/query-generator.js index 12f5e22c0071..cb3e589e1ec3 100755 --- a/lib/dialects/abstract/query-generator.js +++ b/lib/dialects/abstract/query-generator.js @@ -136,12 +136,19 @@ class QueryGenerator { if (this.dialect === 'postgres') { if (options.onConflict && semver.gte(this.sequelize.options.databaseVersion, '9.5.0')) { const constraint = options.conflict.constraint; - const target = options.conflict.target; + const target = Array.isArray(options.conflict.target) ? options.conflict.target : [options.conflict.target]; if (constraint) { valueQuery += ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint) + options.onConflict; } else if (target) { const conflictTarget = target.map(e => this.quoteIdentifier(e)).join(', '); - valueQuery += ' ON CONFLICT (' + conflictTarget + ') ' + options.onConflict; + valueQuery += ' ON CONFLICT (' + conflictTarget + ') '; + + const conflictPredicate = options.conflict.update.where; + if (conflictPredicate) { + valueQuery += 'WHERE ' + this.whereItemsQuery(conflictPredicate, options) + ' '; + } + + valueQuery += + options.onConflict; } valueQuery += ' RETURNING ' + options.returning; } else { @@ -338,21 +345,30 @@ class QueryGenerator { options.updateOnConflict = _.extend({ constraint: '', target: primaryKeyField, - update: allAttributes.filter(value => value !== primaryKeyField) + update: {} }, options.updateOnConflict || {}); + if (!options.updateOnConflict.update.columns) { + options.updateOnConflict.update.columns = allAttributes.filter(value => value !== primaryKeyField); + } const { constraint, target, update } = options.updateOnConflict; let conflict = ''; if (constraint) { conflict = ' ON CONFLICT ON CONSTRAINT ' + this.quoteIdentifier(constraint); } else { - conflict = ' ON CONFLICT (' + this.quoteIdentifier(target) + ')'; + const targetArr = Array.isArray(target) ? target : [target]; + const conflictTarget = targetArr.map(e => this.quoteIdentifier(e)).join(', '); + conflict = ' ON CONFLICT (' + conflictTarget + ')'; + + if (update.where) { + conflict += ' WHERE ' + this.whereItemsQuery(update.where, options); + } } if (update === false) { onConflictKeyUpdate += ' DO NOTHING '; } else { - onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + update.map(attr => { + onConflictKeyUpdate += conflict + ' DO UPDATE SET ' + update.columns.map(attr => { const field = rawAttributes[attr] ? rawAttributes[attr].fieldName : attr; const key = this.quoteIdentifier(field); return key + ' = EXCLUDED.' + key; diff --git a/lib/dialects/postgres/query-generator.js b/lib/dialects/postgres/query-generator.js index b5f38cd3cec1..17d9f8131652 100755 --- a/lib/dialects/postgres/query-generator.js +++ b/lib/dialects/postgres/query-generator.js @@ -388,12 +388,16 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { options.isPgUpsert = true; if (!options.conflict) { options.conflict = { - target: uniqueKeys, - update: getUpdateArray(updateValues, uniqueKeys) + target: uniqueKeys + }; + } + if (!options.conflict.update) { + options.conflict.update = { + columns: getUpdateArray(updateValues, uniqueKeys) }; } - if (options.conflict.update === false) { + if (options.conflict.update.columns === false) { options.onConflict = ' DO NOTHING'; } else { options.conflict.target = options.conflict.target || uniqueKeys; @@ -401,18 +405,18 @@ class PostgresQueryGenerator extends AbstractQueryGenerator { options.conflict.target = [options.conflict.target]; } options.conflict.target = options.conflict.target.map(key => fieldMap[key] || key); - if (!options.conflict.update) { - options.conflict.update = getUpdateArray(updateValues, options.conflict.target); + if (!options.conflict.update.columns) { + options.conflict.update.columns = getUpdateArray(updateValues, options.conflict.target); } if (rawAttributes.updatedAt) { insertValues.updatedAt = new Date(); - if (options.conflict.update.indexOf('updatedAt') === -1) { - options.conflict.update.push('updatedAt'); + if (options.conflict.update.columns.indexOf('updatedAt') === -1) { + options.conflict.update.columns.push('updatedAt'); } } - options.onConflict = 'DO UPDATE SET ' + options.conflict.update.map(key => { + options.onConflict = 'DO UPDATE SET ' + options.conflict.update.columns.map(key => { key = this.quoteIdentifier(key); return key + ' = EXCLUDED.' + key; }).join(', ');