From 589a337b78cc8929c91a152f4878eac28d82dca8 Mon Sep 17 00:00:00 2001 From: SadiJr Date: Wed, 15 Feb 2023 11:03:34 -0300 Subject: [PATCH 01/11] Add VPC billing --- .../main/java/com/cloud/event/EventTypes.java | 4 + .../apache/cloudstack/usage/UsageTypes.java | 2 + .../main/java/com/cloud/usage/UsageVpcVO.java | 130 +++++++++++++++++ .../java/com/cloud/usage/dao/UsageVpcDao.java | 29 ++++ .../com/cloud/usage/dao/UsageVpcDaoImpl.java | 131 ++++++++++++++++++ ...s-between-management-and-usage-context.xml | 1 + ...spring-engine-schema-core-daos-context.xml | 1 + .../META-INF/db/schema-41900to41910.sql | 13 ++ .../presetvariables/PresetVariableHelper.java | 22 +++ .../cloudstack/quota/constant/QuotaTypes.java | 1 + .../apache/cloudstack/quota/dao/VpcDao.java | 23 +++ .../cloudstack/quota/dao/VpcDaoImpl.java | 23 +++ .../PresetVariableHelperTest.java | 1 + .../com/cloud/network/vpc/VpcManagerImpl.java | 8 +- .../com/cloud/usage/UsageManagerImpl.java | 25 ++++ .../cloud/usage/parser/VpcUsageParser.java | 90 ++++++++++++ 16 files changed, 503 insertions(+), 1 deletion(-) create mode 100644 engine/schema/src/main/java/com/cloud/usage/UsageVpcVO.java create mode 100644 engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDao.java create mode 100644 engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDao.java create mode 100644 framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDaoImpl.java create mode 100644 usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index d385fa9ed07f..852a109447c9 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -1219,4 +1219,8 @@ public static Class getEntityClassForEvent(String eventName) { return null; } + + public static boolean isVpcEvent(String eventType) { + return EventTypes.EVENT_VPC_CREATE.equals(eventType) || EventTypes.EVENT_VPC_DELETE.equals(eventType); + } } diff --git a/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java b/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java index 5e0f03ff5830..78af844c0a8f 100644 --- a/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java +++ b/api/src/main/java/org/apache/cloudstack/usage/UsageTypes.java @@ -46,6 +46,7 @@ public class UsageTypes { public static final int VM_SNAPSHOT_ON_PRIMARY = 27; public static final int BACKUP = 28; public static final int BUCKET = 29; + public static final int VPC = 31; public static List listUsageTypes() { List responseList = new ArrayList(); @@ -72,6 +73,7 @@ public static List listUsageTypes() { responseList.add(new UsageTypeResponse(VM_SNAPSHOT_ON_PRIMARY, "VM Snapshot on primary storage usage")); responseList.add(new UsageTypeResponse(BACKUP, "Backup storage usage")); responseList.add(new UsageTypeResponse(BUCKET, "Bucket storage usage")); + responseList.add(new UsageTypeResponse(VPC, "VPC usage")); return responseList; } } diff --git a/engine/schema/src/main/java/com/cloud/usage/UsageVpcVO.java b/engine/schema/src/main/java/com/cloud/usage/UsageVpcVO.java new file mode 100644 index 000000000000..e676b2bc2e98 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/UsageVpcVO.java @@ -0,0 +1,130 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import org.apache.cloudstack.api.InternalIdentity; + +import java.util.Date; + +@Entity +@Table(name = "usage_vpc") +public class UsageVpcVO implements InternalIdentity { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + @Column(name = "id") + private long id; + + @Column(name = "vpc_id") + private long vpcId; + + @Column(name = "zone_id") + private long zoneId; + + @Column(name = "account_id") + private long accountId; + + @Column(name = "domain_id") + private long domainId; + + + @Column(name = "state") + private String state; + + @Column(name = "created") + @Temporal(value = TemporalType.TIMESTAMP) + private Date created = null; + + @Column(name = "removed") + @Temporal(value = TemporalType.TIMESTAMP) + private Date removed = null; + + protected UsageVpcVO(){} + + public UsageVpcVO(long id, long vpcId, long zoneId, long accountId, long domainId, String state, Date created, Date removed) { + this.id = id; + this.vpcId = vpcId; + this.zoneId = zoneId; + this.domainId = domainId; + this.accountId = accountId; + this.state = state; + this.created = created; + this.removed = removed; + } + public UsageVpcVO(long vpcId, long zoneId, long accountId, long domainId, String state, Date created, Date removed) { + this.vpcId = vpcId; + this.zoneId = zoneId; + this.domainId = domainId; + this.accountId = accountId; + this.state = state; + this.created = created; + this.removed = removed; + } + + @Override + public long getId() { + return id; + } + + public long getZoneId() { + return zoneId; + } + + public long getAccountId() { + return accountId; + } + + public long getDomainId() { + return domainId; + } + + public long getVpcId() { + return vpcId; + } + + public void setVpcId(long vpcId) { + this.vpcId = vpcId; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public Date getCreated() { + return created; + } + + public Date getRemoved() { + return removed; + } + + public void setRemoved(Date removed) { + this.removed = removed; + } +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDao.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDao.java new file mode 100644 index 000000000000..a1514aba4cad --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDao.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.dao; + +import com.cloud.usage.UsageVpcVO; +import com.cloud.utils.db.GenericDao; + +import java.util.Date; +import java.util.List; + +public interface UsageVpcDao extends GenericDao { + void update(UsageVpcVO usage); + void remove(long vpcId, Date removed); + List getUsageRecords(Long accountId, Date startDate, Date endDate); +} diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java new file mode 100644 index 000000000000..836705e3b7c3 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.dao; + +import com.cloud.network.vpc.Vpc; +import com.cloud.usage.UsageVpcVO; +import com.cloud.utils.DateUtil; +import com.cloud.utils.db.GenericDaoBase; +import com.cloud.utils.db.SearchCriteria; +import com.cloud.utils.db.TransactionLegacy; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +@Component +public class UsageVpcDaoImpl extends GenericDaoBase implements UsageVpcDao { + private static final Logger LOGGER = Logger.getLogger(UsageVpcDaoImpl.class); + protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT id, vpc_id, zone_id, account_id, domain_id, state, created, removed FROM usage_vpc WHERE " + + " account_id = ? AND ((removed IS NULL AND created <= ?) OR (created BETWEEN ? AND ?) OR (removed BETWEEN ? AND ?) " + + " OR ((created <= ?) AND (removed >= ?)))"; + + @Override + public void update(UsageVpcVO usage) { + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + try { + SearchCriteria sc = this.createSearchCriteria(); + sc.addAnd("vpcId", SearchCriteria.Op.EQ, usage.getVpcId()); + sc.addAnd("created", SearchCriteria.Op.EQ, usage.getCreated()); + UsageVpcVO vo = findOneBy(sc); + if (vo != null) { + vo.setRemoved(usage.getRemoved()); + update(vo.getId(), vo); + } + } catch (final Exception e) { + LOGGER.error(String.format("Error updating backup metrics due to [%s].", e.getMessage()), e); + txn.rollback(); + } finally { + txn.close(); + } + } + + @Override + public void remove(long vpcId, Date removed) { + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + try { + SearchCriteria sc = this.createSearchCriteria(); + sc.addAnd("vpcId", SearchCriteria.Op.EQ, vpcId); + sc.addAnd("removed", SearchCriteria.Op.NULL); + UsageVpcVO vo = findOneBy(sc); + if (vo != null) { + vo.setRemoved(removed); + vo.setState(Vpc.State.Inactive.name()); + update(vo.getId(), vo); + } + } catch (final Exception e) { + txn.rollback(); + LOGGER.error(String.format("Error updating usage of VPC due to [%s].", e.getMessage()), e); + } finally { + txn.close(); + } + } + + @Override + public List getUsageRecords(Long accountId, Date startDate, Date endDate) { + List usageRecords = new ArrayList<>(); + TransactionLegacy txn = TransactionLegacy.open(TransactionLegacy.USAGE_DB); + PreparedStatement pstmt; + try { + int i = 1; + pstmt = txn.prepareAutoCloseStatement(GET_USAGE_RECORDS_BY_ACCOUNT); + pstmt.setLong(i++, accountId); + + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), endDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), startDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), endDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), startDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), endDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), startDate)); + pstmt.setString(i++, DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), endDate)); + + ResultSet rs = pstmt.executeQuery(); + while (rs.next()) { + long id = rs.getLong(1); + long vpcId = rs.getLong(2); + long zoneId = rs.getLong(3); + long acctId = rs.getLong(4); + long domId = rs.getLong(5); + String stateTS = rs.getString(6); + Date createdDate = null; + Date removedDate = null; + String createdTS = rs.getString(7); + String removedTS = rs.getString(8); + + if (createdTS != null) { + createdDate = DateUtil.parseDateString(s_gmtTimeZone, createdTS); + } + if (removedTS != null) { + removedDate = DateUtil.parseDateString(s_gmtTimeZone, removedTS); + } + usageRecords.add(new UsageVpcVO(id, vpcId, zoneId, acctId, domId, stateTS, createdDate, removedDate)); + } + } catch (Exception e) { + txn.rollback(); + LOGGER.warn("Error getting VM backup usage records", e); + } finally { + txn.close(); + } + + return usageRecords; + } +} diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml index 0c46c5ff9341..368d17e831d0 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-common-daos-between-management-and-usage-context.xml @@ -67,6 +67,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml index 5d9583831610..62ef8645a7e1 100644 --- a/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml +++ b/engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml @@ -203,6 +203,7 @@ + diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql index e704d61d70c1..b961c76979e4 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql @@ -30,3 +30,16 @@ UPDATE cloud_usage.quota_tariff SET usage_unit = 'IOPS', updated_on = NOW() WHERE effective_on = '2010-05-04 00:00:00' AND name IN ('VM_DISK_IO_READ', 'VM_DISK_IO_WRITE'); + +-- PR #7235 - [Usage] Create VPC billing +CREATE TABLE IF NOT EXISTS `cloud_usage`.`usage_vpc` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + `vpc_id` bigint(20) unsigned NOT NULL, + `zone_id` bigint(20) unsigned NOT NULL, + `account_id` bigint(20) unsigned NOT NULL, + `domain_id` bigint(20) unsigned NOT NULL, + `state` varchar(100) DEFAULT NULL, + `created` datetime NOT NULL, + `removed` datetime DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB CHARSET=utf8; diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java index 9723d3e5899f..95d28ac24197 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelper.java @@ -26,6 +26,7 @@ import java.util.stream.Collectors; import com.cloud.host.HostTagVO; +import com.cloud.network.vpc.VpcVO; import javax.inject.Inject; import com.cloud.hypervisor.Hypervisor; @@ -37,6 +38,7 @@ import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; import org.apache.cloudstack.quota.constant.QuotaTypes; import org.apache.cloudstack.quota.dao.VmTemplateDao; +import org.apache.cloudstack.quota.dao.VpcDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreDao; import org.apache.cloudstack.storage.datastore.db.ImageStoreVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -170,6 +172,10 @@ public class PresetVariableHelper { @Inject BackupOfferingDao backupOfferingDao; + @Inject + VpcDao vpcDao; + + protected boolean backupSnapshotAfterTakingSnapshot = SnapshotInfo.BackupSnapshotAfterTakingSnapshot.value(); private List runningAndAllocatedVmUsageTypes = Arrays.asList(UsageTypes.RUNNING_VM, UsageTypes.ALLOCATED_VM); @@ -273,6 +279,7 @@ protected Value getPresetVariableValue(UsageVO usageRecord) { loadPresetVariableValueForNetworkOffering(usageRecord, value); loadPresetVariableValueForVmSnapshot(usageRecord, value); loadPresetVariableValueForBackup(usageRecord, value); + loadPresetVariableValueForVpc(usageRecord, value); return value; } @@ -689,6 +696,21 @@ protected BackupOffering getPresetVariableValueBackupOffering(Long offeringId) { return backupOffering; } + protected void loadPresetVariableValueForVpc(UsageVO usageRecord, Value value) { + int usageType = usageRecord.getUsageType(); + if (usageType != QuotaTypes.VPC) { + logNotLoadingMessageInTrace("VPC", usageType); + return; + } + + Long vpcId = usageRecord.getUsageId(); + VpcVO vpc = vpcDao.findByIdIncludingRemoved(vpcId); + validateIfObjectIsNull(vpc, vpcId, "VPC"); + + value.setId(vpc.getUuid()); + value.setName(vpc.getName()); + } + /** * Throws a {@link CloudRuntimeException} if the object is null; */ diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java index 6f19fa95f1bc..0c6ff82a4f51 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/constant/QuotaTypes.java @@ -56,6 +56,7 @@ public class QuotaTypes extends UsageTypes { quotaTypeList.put(VM_SNAPSHOT_ON_PRIMARY, new QuotaTypes(VM_SNAPSHOT_ON_PRIMARY, "VM_SNAPSHOT_ON_PRIMARY", UsageUnitTypes.GB_MONTH.toString(), "VM Snapshot primary storage usage")); quotaTypeList.put(BACKUP, new QuotaTypes(BACKUP, "BACKUP", UsageUnitTypes.GB_MONTH.toString(), "Backup storage usage")); quotaTypeList.put(BUCKET, new QuotaTypes(BUCKET, "BUCKET", UsageUnitTypes.GB_MONTH.toString(), "Object Store bucket usage")); + quotaTypeList.put(VPC, new QuotaTypes(VPC, "VPC", UsageUnitTypes.COMPUTE_MONTH.toString(), "VPC usage")); quotaTypeMap = Collections.unmodifiableMap(quotaTypeList); } diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDao.java new file mode 100644 index 000000000000..e6f810628bf1 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDao.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.network.vpc.VpcVO; +import com.cloud.utils.db.GenericDao; + +public interface VpcDao extends GenericDao { +} \ No newline at end of file diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDaoImpl.java new file mode 100644 index 000000000000..84024b308972 --- /dev/null +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDaoImpl.java @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.quota.dao; + +import com.cloud.network.vpc.VpcVO; +import com.cloud.utils.db.GenericDaoBase; + +public class VpcDaoImpl extends GenericDaoBase implements VpcDao { +} \ No newline at end of file diff --git a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java index b973d1145c30..f7492590196b 100644 --- a/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java +++ b/framework/quota/src/test/java/org/apache/cloudstack/quota/activationrule/presetvariables/PresetVariableHelperTest.java @@ -471,6 +471,7 @@ public void getPresetVariableValueTestSetFieldsAndReturnObject() { Mockito.doNothing().when(presetVariableHelperSpy).loadPresetVariableValueForNetworkOffering(Mockito.any(UsageVO.class), Mockito.any(Value.class)); Mockito.doNothing().when(presetVariableHelperSpy).loadPresetVariableValueForVmSnapshot(Mockito.any(UsageVO.class), Mockito.any(Value.class)); Mockito.doNothing().when(presetVariableHelperSpy).loadPresetVariableValueForBackup(Mockito.any(UsageVO.class), Mockito.any(Value.class)); + Mockito.doNothing().when(presetVariableHelperSpy).loadPresetVariableValueForVpc(Mockito.any(UsageVO.class), Mockito.any(Value.class)); Value result = presetVariableHelperSpy.getPresetVariableValue(usageVoMock); diff --git a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java index 341a3b81b423..58098ee38327 100644 --- a/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java +++ b/server/src/main/java/com/cloud/network/vpc/VpcManagerImpl.java @@ -37,6 +37,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import com.cloud.event.UsageEventUtils; import javax.annotation.PostConstruct; import javax.inject.Inject; import javax.naming.ConfigurationException; @@ -1147,7 +1148,7 @@ protected Vpc createVpc(final Boolean displayVpc, final VpcVO vpc) { + "and the hyphen ('-'); can't start or end with \"-\""); } - return Transaction.execute(new TransactionCallback() { + VpcVO vpcVO = Transaction.execute(new TransactionCallback() { @Override public VpcVO doInTransaction(final TransactionStatus status) { final VpcVO persistedVpc = vpcDao.persist(vpc, finalizeServicesAndProvidersForVpc(vpc.getZoneId(), vpc.getVpcOfferingId())); @@ -1157,6 +1158,10 @@ public VpcVO doInTransaction(final TransactionStatus status) { return persistedVpc; } }); + if (vpcVO != null) { + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VPC_CREATE, vpcVO.getAccountId(), vpcVO.getZoneId(), vpcVO.getId(), vpcVO.getName(), Vpc.class.getName(), vpcVO.getUuid(), vpcVO.isDisplay()); + } + return vpcVO; } private Map> finalizeServicesAndProvidersForVpc(final long zoneId, final long offeringId) { @@ -1255,6 +1260,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { // executed successfully if (vpcDao.remove(vpc.getId())) { s_logger.debug("Vpc " + vpc + " is destroyed successfully"); + UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VPC_DELETE, vpc.getAccountId(), vpc.getZoneId(), vpc.getId(), vpc.getName(), Vpc.class.getName(), vpc.getUuid(), vpc.isDisplay()); return true; } else { s_logger.warn("Vpc " + vpc + " failed to destroy"); diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index a001ffe147c3..4d75eeb65646 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -32,6 +32,9 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import com.cloud.network.vpc.Vpc; +import com.cloud.usage.dao.UsageVpcDao; +import com.cloud.usage.parser.VpcUsageParser; import javax.inject.Inject; import javax.naming.ConfigurationException; import javax.persistence.EntityExistsException; @@ -170,6 +173,9 @@ public class UsageManagerImpl extends ManagerBase implements UsageManager, Runna @Inject private BucketStatisticsDao _bucketStatisticsDao; + @Inject + private UsageVpcDao usageVpcDao; + private String _version = null; private final Calendar _jobExecTime = Calendar.getInstance(); private int _aggregationDuration = 0; @@ -1031,6 +1037,10 @@ private boolean parseHelperTables(AccountVO account, Date currentStartDate, Date s_logger.debug("Bucket usage successfully parsed? " + parsed + " (for account: " + account.getAccountName() + ", id: " + account.getId() + ")"); } } + parsed = VpcUsageParser.parse(account, currentStartDate, currentEndDate); + if (!parsed) { + s_logger.debug(String.format("VPC usage failed to parse for account [%s].", account)); + } return parsed; } @@ -1065,6 +1075,8 @@ private void createHelperRecord(UsageEventVO event) { createVmSnapshotOnPrimaryEvent(event); } else if (isBackupEvent(eventType)) { createBackupEvent(event); + } else if (EventTypes.isVpcEvent(eventType)) { + handleVpcEvent(event); } } catch (EntityExistsException e) { s_logger.warn(String.format("Failed to create usage event id: %d type: %s due to %s", event.getId(), eventType, e.getMessage()), e); @@ -2110,6 +2122,19 @@ private void createBackupEvent(final UsageEventVO event) { } } + private void handleVpcEvent(UsageEventVO event) { + Account account = _accountDao.findByIdIncludingRemoved(event.getAccountId()); + long domainId = account.getDomainId(); + if (EventTypes.EVENT_VPC_DELETE.equals(event.getType())) { + usageVpcDao.remove(event.getResourceId(), event.getCreateDate()); + } else if (EventTypes.EVENT_VPC_CREATE.equals(event.getType())) { + UsageVpcVO usageVPCVO = new UsageVpcVO(event.getResourceId(), event.getZoneId(), event.getAccountId(), domainId, Vpc.State.Enabled.name(), event.getCreateDate(), null); + usageVpcDao.persist(usageVPCVO); + } else { + s_logger.error(String.format("Unknown event type [%s] in VPC event parser. Skipping it.", event.getType())); + } + } + private class Heartbeat extends ManagedContextRunnable { @Override protected void runInContext() { diff --git a/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java new file mode 100644 index 000000000000..06ffe55b8e28 --- /dev/null +++ b/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.usage.parser; + +import com.cloud.usage.UsageVpcVO; +import com.cloud.usage.dao.UsageDao; +import com.cloud.usage.UsageVO; +import com.cloud.usage.dao.UsageVpcDao; +import com.cloud.user.AccountVO; +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import org.apache.cloudstack.usage.UsageTypes; +import org.springframework.stereotype.Component; + +import java.text.DecimalFormat; +import java.util.Date; +import java.util.List; +import org.apache.log4j.Logger; + +@Component +public class VpcUsageParser { + private static final Logger LOGGER = Logger.getLogger(VpcUsageParser.class.getName()); + + @Inject + private UsageVpcDao vpcDao; + @Inject + private UsageDao usageDao; + + private static UsageDao s_usageDao; + private static UsageVpcDao s_usageVpcDao; + @PostConstruct + void init() { + s_usageDao = usageDao; + s_usageVpcDao = vpcDao; + } + + public static boolean parse(AccountVO account, Date startDate, Date endDate) { + LOGGER.debug(String.format("Parsing all VPC usage events for account [%s].", account.getId())); + if ((endDate == null) || endDate.after(new Date())) { + endDate = new Date(); + } + + final List usageVPCs = s_usageVpcDao.getUsageRecords(account.getId(), startDate, endDate); + if (usageVPCs == null || usageVPCs.isEmpty()) { + LOGGER.debug(String.format("Cannot find any VPC usage for account [%s] in period between [%s] and [%s].", account, startDate, endDate)); + return true; + } + + for (final UsageVpcVO usageVPC : usageVPCs) { + Long zoneId = usageVPC.getZoneId(); + Date createdDate = usageVPC.getCreated(); + Date removedDate = usageVPC.getRemoved(); + if (createdDate.before(startDate)) { + createdDate = startDate; + } + + if (removedDate == null || removedDate.after(endDate)) { + removedDate = endDate; + } + + final long duration = (removedDate.getTime() - createdDate.getTime()) + 1; + final float usage = duration / 1000f / 60f / 60f; + DecimalFormat dFormat = new DecimalFormat("#.######"); + String usageDisplay = dFormat.format(usage); + + String description = String.format("VPC usage for VPC ID: %d", usageVPC.getVpcId()); + UsageVO usageRecord = + new UsageVO(zoneId, account.getAccountId(), account.getDomainId(), description, usageDisplay + " Hrs", + UsageTypes.VPC, (double) usage, null, null, null, null, usageVPC.getVpcId(), + (long)0, null, startDate, endDate); + s_usageDao.persist(usageRecord); + } + + return true; + } +} From e917c2adf35b54ea2b717028a214872bc7f837cb Mon Sep 17 00:00:00 2001 From: SadiJr Date: Tue, 21 Feb 2023 09:44:41 -0300 Subject: [PATCH 02/11] Address stephankruggg reviews --- .../src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java index 836705e3b7c3..a42b96486a54 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java @@ -52,7 +52,7 @@ public void update(UsageVpcVO usage) { update(vo.getId(), vo); } } catch (final Exception e) { - LOGGER.error(String.format("Error updating backup metrics due to [%s].", e.getMessage()), e); + LOGGER.error(String.format("Error updating usage of VPC due to [%s].", e.getMessage()), e); txn.rollback(); } finally { txn.close(); @@ -121,7 +121,7 @@ public List getUsageRecords(Long accountId, Date startDate, Date end } } catch (Exception e) { txn.rollback(); - LOGGER.warn("Error getting VM backup usage records", e); + LOGGER.warn("Error getting VPC usage records", e); } finally { txn.close(); } From 7b94977894cf85f428a5926efa503c36ee975706 Mon Sep 17 00:00:00 2001 From: SadiJr Date: Thu, 18 May 2023 14:34:55 -0300 Subject: [PATCH 03/11] Address Daan reviews, changing script upgrade path --- .../src/main/resources/META-INF/db/schema-41720to41800.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql index c51d5a43045e..2efdc9945a51 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql @@ -1561,7 +1561,6 @@ CREATE VIEW `cloud`.`user_view` AS DELETE FROM `cloud`.`snapshot_store_ref` WHERE store_role = "Primary" AND store_id IN (SELECT id FROM storage_pool WHERE removed IS NOT NULL); - -- Change usage of VM_DISK_IO_WRITE to use right usage_type UPDATE `cloud_usage`.`cloud_usage` From 00ce971fefc86388388f52af7fd541525a9d1ab7 Mon Sep 17 00:00:00 2001 From: SadiJr Date: Tue, 22 Aug 2023 16:23:03 -0300 Subject: [PATCH 04/11] Readd line --- .../src/main/resources/META-INF/db/schema-41720to41800.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql index 2efdc9945a51..c51d5a43045e 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41720to41800.sql @@ -1561,6 +1561,7 @@ CREATE VIEW `cloud`.`user_view` AS DELETE FROM `cloud`.`snapshot_store_ref` WHERE store_role = "Primary" AND store_id IN (SELECT id FROM storage_pool WHERE removed IS NOT NULL); + -- Change usage of VM_DISK_IO_WRITE to use right usage_type UPDATE `cloud_usage`.`cloud_usage` From 5b8122c6398fb716c0b42fe00e254b2bdc70c93c Mon Sep 17 00:00:00 2001 From: SadiJr Date: Mon, 18 Sep 2023 15:27:17 -0300 Subject: [PATCH 05/11] Fix end of files --- .../src/main/java/org/apache/cloudstack/quota/dao/VpcDao.java | 2 +- .../main/java/org/apache/cloudstack/quota/dao/VpcDaoImpl.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDao.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDao.java index e6f810628bf1..a3ce164ce6a3 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDao.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDao.java @@ -20,4 +20,4 @@ import com.cloud.utils.db.GenericDao; public interface VpcDao extends GenericDao { -} \ No newline at end of file +} diff --git a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDaoImpl.java b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDaoImpl.java index 84024b308972..10d7c4bd82ff 100644 --- a/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDaoImpl.java +++ b/framework/quota/src/main/java/org/apache/cloudstack/quota/dao/VpcDaoImpl.java @@ -20,4 +20,4 @@ import com.cloud.utils.db.GenericDaoBase; public class VpcDaoImpl extends GenericDaoBase implements VpcDao { -} \ No newline at end of file +} From b301b1db88b3ea63b287606229640b4d0836dbf5 Mon Sep 17 00:00:00 2001 From: SadiJr Date: Tue, 12 Dec 2023 10:39:43 -0300 Subject: [PATCH 06/11] Address Daan review. --- .../src/main/resources/META-INF/db/schema-41900to41910.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql index b961c76979e4..c47df2d83942 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql @@ -43,3 +43,5 @@ CREATE TABLE IF NOT EXISTS `cloud_usage`.`usage_vpc` ( `removed` datetime DEFAULT NULL, PRIMARY KEY (`id`) ) ENGINE=InnoDB CHARSET=utf8; + +ALTER TABLE `cloud_usage`.`cloud_usage` ADD state VARCHAR(100) DEFAULT NULL; From fbf62cf4e258c200c04805dad8b267a8d215fa4e Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Fri, 1 Mar 2024 10:19:41 -0300 Subject: [PATCH 07/11] Change logger to log4j2 apis --- .../main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java | 8 +++----- .../main/java/com/cloud/usage/parser/VpcUsageParser.java | 9 +++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java index a42b96486a54..152d249e5818 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java @@ -22,7 +22,6 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; -import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import java.sql.PreparedStatement; @@ -34,7 +33,6 @@ @Component public class UsageVpcDaoImpl extends GenericDaoBase implements UsageVpcDao { - private static final Logger LOGGER = Logger.getLogger(UsageVpcDaoImpl.class); protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT id, vpc_id, zone_id, account_id, domain_id, state, created, removed FROM usage_vpc WHERE " + " account_id = ? AND ((removed IS NULL AND created <= ?) OR (created BETWEEN ? AND ?) OR (removed BETWEEN ? AND ?) " + " OR ((created <= ?) AND (removed >= ?)))"; @@ -52,7 +50,7 @@ public void update(UsageVpcVO usage) { update(vo.getId(), vo); } } catch (final Exception e) { - LOGGER.error(String.format("Error updating usage of VPC due to [%s].", e.getMessage()), e); + logger.error("Error updating usage of VPC due to [{}].", e.getMessage(), e); txn.rollback(); } finally { txn.close(); @@ -74,7 +72,7 @@ public void remove(long vpcId, Date removed) { } } catch (final Exception e) { txn.rollback(); - LOGGER.error(String.format("Error updating usage of VPC due to [%s].", e.getMessage()), e); + logger.error("Error updating usage of VPC due to [{}].", e.getMessage(), e); } finally { txn.close(); } @@ -121,7 +119,7 @@ public List getUsageRecords(Long accountId, Date startDate, Date end } } catch (Exception e) { txn.rollback(); - LOGGER.warn("Error getting VPC usage records", e); + logger.warn("Error getting VPC usage records", e); } finally { txn.close(); } diff --git a/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java index 06ffe55b8e28..388e75e6f611 100644 --- a/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java @@ -24,16 +24,17 @@ import javax.annotation.PostConstruct; import javax.inject.Inject; import org.apache.cloudstack.usage.UsageTypes; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; import java.text.DecimalFormat; import java.util.Date; import java.util.List; -import org.apache.log4j.Logger; @Component public class VpcUsageParser { - private static final Logger LOGGER = Logger.getLogger(VpcUsageParser.class.getName()); + protected static Logger LOGGER = LogManager.getLogger(BackupUsageParser.class); @Inject private UsageVpcDao vpcDao; @@ -49,14 +50,14 @@ void init() { } public static boolean parse(AccountVO account, Date startDate, Date endDate) { - LOGGER.debug(String.format("Parsing all VPC usage events for account [%s].", account.getId())); + LOGGER.debug("Parsing all VPC usage events for account [{}].", account.getId()); if ((endDate == null) || endDate.after(new Date())) { endDate = new Date(); } final List usageVPCs = s_usageVpcDao.getUsageRecords(account.getId(), startDate, endDate); if (usageVPCs == null || usageVPCs.isEmpty()) { - LOGGER.debug(String.format("Cannot find any VPC usage for account [%s] in period between [%s] and [%s].", account, startDate, endDate)); + LOGGER.debug("Cannot find any VPC usage for account [{}] in period between [{}] and [{}].", account, startDate, endDate); return true; } From 1706157cb199c85bec3635427fe9bb4c6b457a92 Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Fri, 1 Mar 2024 10:39:01 -0300 Subject: [PATCH 08/11] Fix leftovers logs --- usage/src/main/java/com/cloud/usage/UsageManagerImpl.java | 4 ++-- .../src/main/java/com/cloud/usage/parser/VpcUsageParser.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 4d75eeb65646..3ac451928d5d 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -1039,7 +1039,7 @@ private boolean parseHelperTables(AccountVO account, Date currentStartDate, Date } parsed = VpcUsageParser.parse(account, currentStartDate, currentEndDate); if (!parsed) { - s_logger.debug(String.format("VPC usage failed to parse for account [%s].", account)); + logger.debug("VPC usage failed to parse for account [{}].", account); } return parsed; } @@ -2131,7 +2131,7 @@ private void handleVpcEvent(UsageEventVO event) { UsageVpcVO usageVPCVO = new UsageVpcVO(event.getResourceId(), event.getZoneId(), event.getAccountId(), domainId, Vpc.State.Enabled.name(), event.getCreateDate(), null); usageVpcDao.persist(usageVPCVO); } else { - s_logger.error(String.format("Unknown event type [%s] in VPC event parser. Skipping it.", event.getType())); + logger.error("Unknown event type [{}] in VPC event parser. Skipping it.", event.getType()); } } diff --git a/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java index 388e75e6f611..42119277a243 100644 --- a/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java @@ -34,7 +34,7 @@ @Component public class VpcUsageParser { - protected static Logger LOGGER = LogManager.getLogger(BackupUsageParser.class); + protected static Logger LOGGER = LogManager.getLogger(VpcUsageParser.class); @Inject private UsageVpcDao vpcDao; From bb4659fdb630b32419e8da470d4d8e86c1d5d310 Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Mon, 4 Mar 2024 14:23:45 -0300 Subject: [PATCH 09/11] Add logs --- .../main/java/com/cloud/usage/parser/VpcUsageParser.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java index 42119277a243..800caacfaece 100644 --- a/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.usage.parser; +import com.cloud.usage.UsageManagerImpl; import com.cloud.usage.UsageVpcVO; import com.cloud.usage.dao.UsageDao; import com.cloud.usage.UsageVO; @@ -23,6 +24,8 @@ import com.cloud.user.AccountVO; import javax.annotation.PostConstruct; import javax.inject.Inject; + +import com.cloud.utils.DateUtil; import org.apache.cloudstack.usage.UsageTypes; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -78,6 +81,11 @@ public static boolean parse(AccountVO account, Date startDate, Date endDate) { DecimalFormat dFormat = new DecimalFormat("#.######"); String usageDisplay = dFormat.format(usage); + long vpcId = usageVPC.getVpcId(); + LOGGER.debug("Creating VPC usage record with id [{}], usage [{}], startDate [{}], and endDate [{}], for account [{}].", vpcId, usageDisplay, + DateUtil.displayDateInTimezone(UsageManagerImpl.getUsageAggregationTimeZone(), startDate), + DateUtil.displayDateInTimezone(UsageManagerImpl.getUsageAggregationTimeZone(), endDate), account.getId()); + String description = String.format("VPC usage for VPC ID: %d", usageVPC.getVpcId()); UsageVO usageRecord = new UsageVO(zoneId, account.getAccountId(), account.getDomainId(), description, usageDisplay + " Hrs", From 04159b06f57570ebe8d51f189f0454b67b57f34e Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Thu, 21 Mar 2024 11:03:36 -0300 Subject: [PATCH 10/11] Change add column to idempotent procedure --- .../src/main/resources/META-INF/db/schema-41900to41910.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql index c47df2d83942..358cc9d29790 100644 --- a/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql +++ b/engine/schema/src/main/resources/META-INF/db/schema-41900to41910.sql @@ -44,4 +44,4 @@ CREATE TABLE IF NOT EXISTS `cloud_usage`.`usage_vpc` ( PRIMARY KEY (`id`) ) ENGINE=InnoDB CHARSET=utf8; -ALTER TABLE `cloud_usage`.`cloud_usage` ADD state VARCHAR(100) DEFAULT NULL; +CALL `cloud_usage`.`IDEMPOTENT_ADD_COLUMN`('cloud_usage.cloud_usage', 'state', 'VARCHAR(100) DEFAULT NULL'); From 518b79a49595940b95eeaab2f583daec140a40e3 Mon Sep 17 00:00:00 2001 From: Bryan Lima Date: Tue, 26 Mar 2024 11:29:46 -0300 Subject: [PATCH 11/11] Fix logs --- .../com/cloud/usage/dao/UsageVpcDaoImpl.java | 8 +++++--- .../java/com/cloud/usage/UsageManagerImpl.java | 4 ++-- .../com/cloud/usage/parser/VpcUsageParser.java | 17 ++++++----------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java index 152d249e5818..a42b96486a54 100644 --- a/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/usage/dao/UsageVpcDaoImpl.java @@ -22,6 +22,7 @@ import com.cloud.utils.db.GenericDaoBase; import com.cloud.utils.db.SearchCriteria; import com.cloud.utils.db.TransactionLegacy; +import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import java.sql.PreparedStatement; @@ -33,6 +34,7 @@ @Component public class UsageVpcDaoImpl extends GenericDaoBase implements UsageVpcDao { + private static final Logger LOGGER = Logger.getLogger(UsageVpcDaoImpl.class); protected static final String GET_USAGE_RECORDS_BY_ACCOUNT = "SELECT id, vpc_id, zone_id, account_id, domain_id, state, created, removed FROM usage_vpc WHERE " + " account_id = ? AND ((removed IS NULL AND created <= ?) OR (created BETWEEN ? AND ?) OR (removed BETWEEN ? AND ?) " + " OR ((created <= ?) AND (removed >= ?)))"; @@ -50,7 +52,7 @@ public void update(UsageVpcVO usage) { update(vo.getId(), vo); } } catch (final Exception e) { - logger.error("Error updating usage of VPC due to [{}].", e.getMessage(), e); + LOGGER.error(String.format("Error updating usage of VPC due to [%s].", e.getMessage()), e); txn.rollback(); } finally { txn.close(); @@ -72,7 +74,7 @@ public void remove(long vpcId, Date removed) { } } catch (final Exception e) { txn.rollback(); - logger.error("Error updating usage of VPC due to [{}].", e.getMessage(), e); + LOGGER.error(String.format("Error updating usage of VPC due to [%s].", e.getMessage()), e); } finally { txn.close(); } @@ -119,7 +121,7 @@ public List getUsageRecords(Long accountId, Date startDate, Date end } } catch (Exception e) { txn.rollback(); - logger.warn("Error getting VPC usage records", e); + LOGGER.warn("Error getting VPC usage records", e); } finally { txn.close(); } diff --git a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java index 3ac451928d5d..4d75eeb65646 100644 --- a/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java +++ b/usage/src/main/java/com/cloud/usage/UsageManagerImpl.java @@ -1039,7 +1039,7 @@ private boolean parseHelperTables(AccountVO account, Date currentStartDate, Date } parsed = VpcUsageParser.parse(account, currentStartDate, currentEndDate); if (!parsed) { - logger.debug("VPC usage failed to parse for account [{}].", account); + s_logger.debug(String.format("VPC usage failed to parse for account [%s].", account)); } return parsed; } @@ -2131,7 +2131,7 @@ private void handleVpcEvent(UsageEventVO event) { UsageVpcVO usageVPCVO = new UsageVpcVO(event.getResourceId(), event.getZoneId(), event.getAccountId(), domainId, Vpc.State.Enabled.name(), event.getCreateDate(), null); usageVpcDao.persist(usageVPCVO); } else { - logger.error("Unknown event type [{}] in VPC event parser. Skipping it.", event.getType()); + s_logger.error(String.format("Unknown event type [%s] in VPC event parser. Skipping it.", event.getType())); } } diff --git a/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java b/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java index 800caacfaece..8fff2efde943 100644 --- a/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java +++ b/usage/src/main/java/com/cloud/usage/parser/VpcUsageParser.java @@ -16,7 +16,6 @@ // under the License. package com.cloud.usage.parser; -import com.cloud.usage.UsageManagerImpl; import com.cloud.usage.UsageVpcVO; import com.cloud.usage.dao.UsageDao; import com.cloud.usage.UsageVO; @@ -24,20 +23,17 @@ import com.cloud.user.AccountVO; import javax.annotation.PostConstruct; import javax.inject.Inject; - -import com.cloud.utils.DateUtil; import org.apache.cloudstack.usage.UsageTypes; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.springframework.stereotype.Component; import java.text.DecimalFormat; import java.util.Date; import java.util.List; +import org.apache.log4j.Logger; @Component public class VpcUsageParser { - protected static Logger LOGGER = LogManager.getLogger(VpcUsageParser.class); + private static final Logger LOGGER = Logger.getLogger(VpcUsageParser.class.getName()); @Inject private UsageVpcDao vpcDao; @@ -53,14 +49,14 @@ void init() { } public static boolean parse(AccountVO account, Date startDate, Date endDate) { - LOGGER.debug("Parsing all VPC usage events for account [{}].", account.getId()); + LOGGER.debug(String.format("Parsing all VPC usage events for account [%s].", account.getId())); if ((endDate == null) || endDate.after(new Date())) { endDate = new Date(); } final List usageVPCs = s_usageVpcDao.getUsageRecords(account.getId(), startDate, endDate); if (usageVPCs == null || usageVPCs.isEmpty()) { - LOGGER.debug("Cannot find any VPC usage for account [{}] in period between [{}] and [{}].", account, startDate, endDate); + LOGGER.debug(String.format("Cannot find any VPC usage for account [%s] in period between [%s] and [%s].", account, startDate, endDate)); return true; } @@ -82,9 +78,8 @@ public static boolean parse(AccountVO account, Date startDate, Date endDate) { String usageDisplay = dFormat.format(usage); long vpcId = usageVPC.getVpcId(); - LOGGER.debug("Creating VPC usage record with id [{}], usage [{}], startDate [{}], and endDate [{}], for account [{}].", vpcId, usageDisplay, - DateUtil.displayDateInTimezone(UsageManagerImpl.getUsageAggregationTimeZone(), startDate), - DateUtil.displayDateInTimezone(UsageManagerImpl.getUsageAggregationTimeZone(), endDate), account.getId()); + LOGGER.debug(String.format("Creating VPC usage record with id [%s], usage [%s], startDate [%s], and endDate [%s], for account [%s].", + vpcId, usageDisplay, startDate, endDate, account.getId())); String description = String.format("VPC usage for VPC ID: %d", usageVPC.getVpcId()); UsageVO usageRecord =