diff --git a/.env b/.env index 16152ac..14df861 100644 --- a/.env +++ b/.env @@ -14,7 +14,17 @@ MAIL_DB_URL=jdbc:mariadb://host.docker.internal:3306/mailing?autoReconnect=true& MAIL_DB_USERNM=mcmpcostopti MAIL_DB_PW=0000 # CostCollector -COST_COLLECT_CRON_SCHEDULE=0 30 0 * * ? +COST_COLLECT_UNUSED_CRON_SCHEDULE=0 30 0 * * ? +COST_COLLECT_CUR_CRON_SCHEDULE=0 0 0,6 * * ? +AWS_CUR_EXPORT_NAME=MCMP-CostOpti +AWS_CUR_EXPORT_PATH_PREFIX=mcmp-costopti +AWS_ACCESS_KEY_ID=accesskey +AWS_SECRET_ACCESS_KEY=secretkey # CostProcessor -COST_PROCESS_CRON_SCHEDULE=0 45 * * * ? +COST_PROCESS_UNUSED_CRON_SCHEDULE=0 45 * * * ? +COST_PROCESS_ABNORMAL_CRON_SCHEDULE=0 0 1,7 * * ? COST_SELECTOR_URL=http://costselector:8083 +ALARM_URL=http://localhost:9000 +# AssetCollector +ASSET_MONITORING_SERVER=http://asset.mornitor.server:8080 +ASSET_COLLECT_BATCH_CRON_SCHEDULE=0 0 0 * * ? diff --git a/AlarmService/src/main/java/com/mcmp/slack_demo/common/CommonController.java b/AlarmService/src/main/java/com/mcmp/slack_demo/common/CommonController.java index b3e5e4b..d2f04c0 100644 --- a/AlarmService/src/main/java/com/mcmp/slack_demo/common/CommonController.java +++ b/AlarmService/src/main/java/com/mcmp/slack_demo/common/CommonController.java @@ -4,6 +4,7 @@ import com.mcmp.slack_demo.common.model.costOpti.CostOptiAlarmReqModel; import com.mcmp.slack_demo.mail.model.MailMessage; import com.mcmp.slack_demo.mail.service.MailService; +import com.mcmp.slack_demo.slack.api_client.SlackACService; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import org.springframework.http.ResponseEntity; @@ -19,6 +20,9 @@ public class CommonController { @Autowired private MailService mailService; + @Autowired + private SlackACService slackService; + @PostMapping("/sendOptiAlarmMail") public ResponseEntity sendOptiAlarmMail(@RequestBody CostOptiAlarmReqModel reqModel){ CommonResultModel result = new CommonResultModel(); @@ -29,6 +33,7 @@ public ResponseEntity sendOptiAlarmMail(@RequestBody CostOpti mailService.sendEmail(reqModel, null); break; case "slack": + slackService.sendSlack(reqModel); break; } } diff --git a/AlarmService/src/main/java/com/mcmp/slack_demo/common/dao/CommonDao.java b/AlarmService/src/main/java/com/mcmp/slack_demo/common/dao/CommonDao.java index 5527d7a..0687711 100644 --- a/AlarmService/src/main/java/com/mcmp/slack_demo/common/dao/CommonDao.java +++ b/AlarmService/src/main/java/com/mcmp/slack_demo/common/dao/CommonDao.java @@ -1,6 +1,8 @@ package com.mcmp.slack_demo.common.dao; +import com.mcmp.slack_demo.common.model.costOpti.CostOptiAlarmReqModel; import com.mcmp.slack_demo.mail.model.SendMailFormModel; +import com.mcmp.slack_demo.slack.model.SendSlackFormModel; import org.mybatis.spring.SqlSessionTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -15,12 +17,23 @@ public class CommonDao { @Qualifier("sqlSessionTemplateHistory") private SqlSessionTemplate sqlSession; + public int getAlertDuplicate(SendMailFormModel model){ + return sqlSession.selectOne("HistorySql.getAlertDuplicate", model); + } + public void insertAlarmHistory(SendMailFormModel model){ sqlSession.insert("HistorySql.insertAlertHistory", model); } - public List getAlarmMailReceivers(Map param){ + public List getAlarmMailReceivers(CostOptiAlarmReqModel param){ return sqlSession.selectList("HistorySql.getAlarmMailReceivers", param); } + public int getSlackDuplicate(SendSlackFormModel model){ + return sqlSession.selectOne("HistorySql.getAlertDuplicate", model); + } + + public void insertSlackHistory(SendSlackFormModel model){ + sqlSession.insert("HistorySql.insertAlertHistory", model); + } } diff --git a/AlarmService/src/main/java/com/mcmp/slack_demo/common/model/costOpti/CostOptiAlarmModel.java b/AlarmService/src/main/java/com/mcmp/slack_demo/common/model/costOpti/CostOptiAlarmModel.java index 39d3d1a..c7c393a 100644 --- a/AlarmService/src/main/java/com/mcmp/slack_demo/common/model/costOpti/CostOptiAlarmModel.java +++ b/AlarmService/src/main/java/com/mcmp/slack_demo/common/model/costOpti/CostOptiAlarmModel.java @@ -10,8 +10,11 @@ public class CostOptiAlarmModel { private String resource_id; private String resource_type; private LocalDateTime occure_time; + private String csp_type; private String account_id; private String urgency; + private String plan; private String note; + private String project_cd; } diff --git a/AlarmService/src/main/java/com/mcmp/slack_demo/common/service/CommonService.java b/AlarmService/src/main/java/com/mcmp/slack_demo/common/service/CommonService.java index 3720271..87a93f4 100644 --- a/AlarmService/src/main/java/com/mcmp/slack_demo/common/service/CommonService.java +++ b/AlarmService/src/main/java/com/mcmp/slack_demo/common/service/CommonService.java @@ -1,7 +1,9 @@ package com.mcmp.slack_demo.common.service; import com.mcmp.slack_demo.common.dao.CommonDao; +import com.mcmp.slack_demo.common.model.costOpti.CostOptiAlarmReqModel; import com.mcmp.slack_demo.mail.model.SendMailFormModel; +import com.mcmp.slack_demo.slack.model.SendSlackFormModel; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -15,11 +17,23 @@ public class CommonService { @Autowired private CommonDao commonDao; + public int getAlertDuplicate(SendMailFormModel model){ + return commonDao.getAlertDuplicate(model); + } + public void insertAlarmHistory(SendMailFormModel model){ commonDao.insertAlarmHistory(model); } - public List getAlarmMailReceivers(Map param){ + public List getAlarmMailReceivers(CostOptiAlarmReqModel param){ return commonDao.getAlarmMailReceivers(param); } + + public int getSlackDuplicate(SendSlackFormModel model){ + return commonDao.getSlackDuplicate(model); + } + + public void insertSlackHistory(SendSlackFormModel model){ + commonDao.insertSlackHistory(model); + } } diff --git a/AlarmService/src/main/java/com/mcmp/slack_demo/mail/model/SendMailFormModel.java b/AlarmService/src/main/java/com/mcmp/slack_demo/mail/model/SendMailFormModel.java index 0c5c44c..150d14a 100644 --- a/AlarmService/src/main/java/com/mcmp/slack_demo/mail/model/SendMailFormModel.java +++ b/AlarmService/src/main/java/com/mcmp/slack_demo/mail/model/SendMailFormModel.java @@ -10,4 +10,5 @@ public class SendMailFormModel extends CostOptiAlarmReqModel { private List to; private String subject; private String message; + private String alarm_impl; } diff --git a/AlarmService/src/main/java/com/mcmp/slack_demo/mail/service/MailService.java b/AlarmService/src/main/java/com/mcmp/slack_demo/mail/service/MailService.java index 3d957b0..4e19d23 100644 --- a/AlarmService/src/main/java/com/mcmp/slack_demo/mail/service/MailService.java +++ b/AlarmService/src/main/java/com/mcmp/slack_demo/mail/service/MailService.java @@ -51,14 +51,22 @@ public class MailService { public void sendEmail(CostOptiAlarmReqModel optiAlarmReqModel, ClassPathResource file){ SendMailFormModel mailFormModel = new SendMailFormModel(); BeanUtils.copyProperties(optiAlarmReqModel, mailFormModel); - mailFormModel.setOccure_time(ZonedDateTime.now(ZoneId.of("UTC")).toLocalDateTime()); + mailFormModel.setOccure_time(ZonedDateTime.now().toLocalDateTime()); + mailFormModel.setAlarm_impl("mail"); + + int checkDuplicateMail = commonService.getAlertDuplicate(mailFormModel); + if(checkDuplicateMail >= 1){ + log.info("############Send OptiAlertEmail Duplicate : {} - {} - {}############", mailFormModel.getEvent_type(), mailFormModel.getResource_id() + , (mailFormModel.getAccount_id() != null ? mailFormModel.getAccount_id() : mailFormModel.getProject_cd())); + return; + } try { JavaMailSender emailSender = mailConfig.getJavaMailSender(); MimeMessage mimeMessage = emailSender.createMimeMessage(); // find mail receiver - mailFormModel.setTo(getAlarmMailReceivers(optiAlarmReqModel.getResource_id(), optiAlarmReqModel.getAccount_id())); + mailFormModel.setTo(getAlarmMailReceivers(optiAlarmReqModel)); String mailMessage = "[MCMP-Notice] Cost Alarm occurred"; switch (optiAlarmReqModel.getEvent_type()){ @@ -66,22 +74,29 @@ public void sendEmail(CostOptiAlarmReqModel optiAlarmReqModel, ClassPathResource mailFormModel.setSubject("[MCMP-Notice] Cost Alarm occurred : Caution Unused Resources"); mailMessage = "MCMP Cost에서 미사용 자원 주의 알람이 발생했습니다." + "

" + - "계정 : " + mailFormModel.getAccount_id() + "
" + - "리소스 ID : " + mailFormModel.getResource_id(); + "CSP : " + mailFormModel.getCsp_type() + "
" + + "리소스 ID : " + mailFormModel.getResource_id() + "
" + + "리소스 Type : " + mailFormModel.getResource_type() + "
" + + "해당 자원이 미사용 자원으로 의심됩니다."; break; case "Abnormal": mailFormModel.setSubject("[MCMP-Notice] Cost Alarm occurred : Warning Abnormal Cost"); mailMessage = "MCMP Cost에서 이상 비용 경고 알람이 발생했습니다." + "

" + - "계정 : " + mailFormModel.getAccount_id() + "
" + - "리소스 ID : " + mailFormModel.getResource_id(); + "CSP : " + mailFormModel.getCsp_type() + "
" + + "제품군 : " + mailFormModel.getResource_type() + "
" + + "이상비용 등급 : " + mailFormModel.getPlan() + "
" + + "이상비용이 발생했습니다. " + mailFormModel.getNote(); break; case "Resize": - mailFormModel.setSubject("[MCMP-Notice] Cost Alarm occurred : Advise Resizing Resources"); + mailFormModel.setSubject("[MCMP-Notice] Cost Alarm occurred : Advise Right Size Resources"); mailMessage = "MCMP Cost에서 자원 최적화 권고 알람이 발생했습니다." + "

" + - "계정 : " + mailFormModel.getAccount_id() + "
" + - "리소스 ID : " + mailFormModel.getResource_id(); + "CSP : " + mailFormModel.getCsp_type() + "
" + + "리소스 ID : " + mailFormModel.getResource_id() + "
" + + "리소스 Type : " + mailFormModel.getResource_type() + "
" + + "추천 Plan : " + mailFormModel.getPlan() + "
" + + mailFormModel.getNote(); break; } @@ -155,14 +170,8 @@ public MailingInfoModel getMailingInfo(){ return mailingDao.getMailingInfo(); } - private List getAlarmMailReceivers(String resource_id, String account_id){ - Map param = Map.of( - "resource_id", resource_id, - "account_id", account_id - ); - + private List getAlarmMailReceivers(CostOptiAlarmReqModel param){ return commonService.getAlarmMailReceivers(param); - } } diff --git a/AlarmService/src/main/java/com/mcmp/slack_demo/slack/api_client/SlackACService.java b/AlarmService/src/main/java/com/mcmp/slack_demo/slack/api_client/SlackACService.java index a190c7b..92bd2bb 100644 --- a/AlarmService/src/main/java/com/mcmp/slack_demo/slack/api_client/SlackACService.java +++ b/AlarmService/src/main/java/com/mcmp/slack_demo/slack/api_client/SlackACService.java @@ -1,18 +1,27 @@ package com.mcmp.slack_demo.slack.api_client; +import com.mcmp.slack_demo.common.model.costOpti.CostOptiAlarmReqModel; +import com.mcmp.slack_demo.common.service.CommonService; import com.mcmp.slack_demo.slack.encryto.TokenService; +import com.mcmp.slack_demo.slack.model.SendSlackFormModel; import com.slack.api.Slack; import com.slack.api.methods.SlackApiException; import com.slack.api.methods.request.chat.ChatPostMessageRequest; import com.slack.api.methods.response.chat.ChatPostMessageResponse; +import com.slack.api.model.Attachment; +import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import lombok.extern.slf4j.Slf4j; import javax.naming.AuthenticationException; import java.io.IOException; +import java.time.ZonedDateTime; +import java.util.Collections; import java.util.Map; @Service +@Slf4j public class SlackACService { private final TokenService tokenService; @@ -20,6 +29,86 @@ public class SlackACService { public SlackACService(TokenService tokenService) { this.tokenService = tokenService; } + @Autowired + private CommonService commonService; + + public void sendSlack(CostOptiAlarmReqModel costOptiAlarmReqModel) throws SlackApiException, IOException { + SendSlackFormModel slackFormModel = new SendSlackFormModel(); + BeanUtils.copyProperties(costOptiAlarmReqModel, slackFormModel); + slackFormModel.setOccure_time(ZonedDateTime.now().toLocalDateTime()); + slackFormModel.setAlarm_impl("slack"); + + int checkDuplicateMail = commonService.getSlackDuplicate(slackFormModel); + if(checkDuplicateMail >= 1){ + log.info("############Send OptiAlertSlack Duplicate : {} - {} - {}############", slackFormModel.getEvent_type(), slackFormModel.getResource_id() + , (slackFormModel.getAccount_id() != null ? slackFormModel.getAccount_id() : slackFormModel.getProject_cd())); + return; + } + + Slack slack = Slack.getInstance(); + String message = "[MCMP-Notice] Cost Alarm occurred"; + switch (costOptiAlarmReqModel.getEvent_type()){ + case "Unused": + slackFormModel.setTitle("[MCMP-Notice] Cost Alarm occurred : Caution Unused Resources"); + message = "MCMP Cost에서 미사용 자원 주의 알람이 발생했습니다." + + "\n\n" + + "CSP : " + slackFormModel.getCsp_type() + "\n" + + "리소스 ID : " + slackFormModel.getResource_id() + "\n" + + "리소스 Type : " + slackFormModel.getResource_type() + "\n" + + "해당 자원이 미사용 자원으로 의심됩니다."; + break; + case "Abnormal": + slackFormModel.setTitle("[MCMP-Notice] Cost Alarm occurred : Warning Abnormal Cost"); + message = "MCMP Cost에서 이상 비용 경고 알람이 발생했습니다." + + "\n\n" + + "CSP : " + slackFormModel.getCsp_type() + "\n" + + "제품군 : " + slackFormModel.getResource_type() + "\n" + + "이상비용 등급 : " + slackFormModel.getPlan() + "\n" + + "이상비용이 발생했습니다. " + slackFormModel.getNote(); + break; + case "Resize": + slackFormModel.setTitle("[MCMP-Notice] Cost Alarm occurred : Advise Right Size Resources"); + message = "MCMP Cost에서 자원 최적화 권고 알람이 발생했습니다." + + "\n\n" + + "CSP : " + slackFormModel.getCsp_type() + "\n" + + "리소스 ID : " + slackFormModel.getResource_id() + "\n" + + "리소스 Type : " + slackFormModel.getResource_type() + "\n" + + "추천 Plan : " + slackFormModel.getPlan() + "\n" + + slackFormModel.getNote(); + break; + } + try { +// Map result = tokenService.retrieveToken(slackFormModel.getAccount_id()); + Map result = tokenService.retrieveToken("test"); + Attachment attachment = Attachment.builder() + .title(slackFormModel.getTitle()) + .text(message) + .color("#36a64f") + .build(); + + ChatPostMessageRequest request = ChatPostMessageRequest.builder() + .channel(result.get("channel")) + .attachments(Collections.singletonList(attachment)) + .build(); + ChatPostMessageResponse response = slack.methods(result.get("token")).chatPostMessage(request); + if (!response.isOk()) { + switch (response.getError()) { + case "invalid_auth": + throw new AuthenticationException("Invalid authentication credentials for Slack."); + case "channel_not_found": + throw new Exception("The specified channel was not found."); + default: + throw new Exception("An error occurred with Slack API: " + response.getError()); + } + } + } catch (IOException | SlackApiException e) { + e.printStackTrace(); + throw e; + } catch (Exception e) { + throw new RuntimeException(e); + } + commonService.insertSlackHistory(slackFormModel); + } public void sendMessage(String userId, String message, String linkUrl, String linkText) throws SlackApiException, IOException { Slack slack = Slack.getInstance(); diff --git a/AlarmService/src/main/java/com/mcmp/slack_demo/slack/model/SendSlackFormModel.java b/AlarmService/src/main/java/com/mcmp/slack_demo/slack/model/SendSlackFormModel.java new file mode 100644 index 0000000..8903832 --- /dev/null +++ b/AlarmService/src/main/java/com/mcmp/slack_demo/slack/model/SendSlackFormModel.java @@ -0,0 +1,11 @@ +package com.mcmp.slack_demo.slack.model; + +import com.mcmp.slack_demo.common.model.costOpti.CostOptiAlarmReqModel; +import lombok.Data; + +@Data +public class SendSlackFormModel extends CostOptiAlarmReqModel { + private String title; + private String message; + private String alarm_impl; +} diff --git a/AlarmService/src/main/resources/mapper/history/History_SQL.xml b/AlarmService/src/main/resources/mapper/history/History_SQL.xml index 2ef5833..5b89d5f 100644 --- a/AlarmService/src/main/resources/mapper/history/History_SQL.xml +++ b/AlarmService/src/main/resources/mapper/history/History_SQL.xml @@ -1,21 +1,94 @@ + + - INSERT INTO alarm_history(event_type, resource_id, resource_type, occure_time, account_id, urgency, note) - VALUES (#{event_type}, #{resource_id}, #{resource_type}, #{occure_time}, #{account_id}, #{urgency}, #{note}) + INSERT INTO alarm_history(event_type, csp_type, resource_id, resource_type, occure_dt, account_id, urgency, plan, note, occure_date, alarm_impl, project_cd) + VALUES (#{event_type} + , #{csp_type} + , #{resource_id} + , #{resource_type} + , #{occure_time} + , #{account_id} + , #{urgency} + , #{plan} + , #{note} + , date(#{occure_time}) + , #{alarm_impl} + , COALESCE(#{project_cd}, 'BLANK_00_')) + ON DUPLICATE KEY UPDATE occure_dt = #{occure_time} + , account_id = #{account_id} + , urgency = #{urgency} + , plan = #{plan} + , note = #{note} - SELECT tcumr.mcmp_mail_receiver AS receiver FROM servicegroup_meta sm JOIN temp_cmp_user_info tcui - ON tcui.mcmp_user_account = sm.csp_account + ON tcui.csp_account_id = sm.csp_account JOIN temp_cmp_user_mail_receiver tcumr ON tcumr.mcmp_user_id = tcui.mcmp_user_id WHERE 1=1 - AND sm.csp_instanceid = #{resource_id} - AND sm.csp_account = #{account_id} + + + AND sm.csp_instanceid = #{resource_id} + + + AND sm.service_cd = #{project_cd} + group by tcumr.mcmp_mail_receiver + + + + + + INSERT INTO alarm_history(event_type, csp_type, resource_id, resource_type, occure_dt, account_id, urgency, plan, note, occure_date, alarm_impl, project_cd) + VALUES (#{event_type} + , #{csp_type} + , #{resource_id} + , #{resource_type} + , #{occure_time} + , #{account_id} + , #{urgency} + , #{plan} + , #{note} + , date(#{occure_time}) + , #{alarm_impl} + , COALESCE(#{project_cd}, 'BLANK_00_')) + ON DUPLICATE KEY UPDATE occure_dt = #{occure_time} + , account_id = #{account_id} + , urgency = #{urgency} + , plan = #{plan} + , note = #{note} + diff --git a/BackEnd/src/main/java/com/mcmp/costbe/alarm/AlarmController.java b/BackEnd/src/main/java/com/mcmp/costbe/alarm/AlarmController.java new file mode 100644 index 0000000..4544702 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/alarm/AlarmController.java @@ -0,0 +1,46 @@ +package com.mcmp.costbe.alarm; + +import com.mcmp.costbe.alarm.model.AlarmHistoryReqModel; +import com.mcmp.costbe.alarm.model.AlarmHistoryRstModel; +import com.mcmp.costbe.alarm.service.AlarmHistoryService; +import com.mcmp.costbe.common.model.ResultModel; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping(path = "/api/v2/alarm") +@Tag(name = "Cost Alarm", description = "Cost Alarm API") +public class AlarmController { + + @Autowired + private AlarmHistoryService alarmHistoryService; + + @PostMapping(path = "/history") + @Operation(summary = "알람 발생 내역 조회", description = "최근 7일간 발생한 최적화 알람을 조회한다.") + @ApiResponses(value={ + @ApiResponse(responseCode = "200", description = "성공", + content = {@Content(schema = @Schema(implementation = AlarmHistoryRstModel.class))}), + @ApiResponse(responseCode = "500", description = "서버 오류", content = {@Content(examples = {})}) + }) + public ResponseEntity getAlarmHistory(@RequestBody AlarmHistoryReqModel req){ + ResultModel result = new ResultModel(); + try { + AlarmHistoryRstModel data = alarmHistoryService.getAlarmHistory(req); + result.setData(data); + } catch (Exception e){ + e.printStackTrace(); + result.setError(500, "Failed to get Alarm History Info"); + } + return ResponseEntity.ok(result); + } +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/alarm/dao/AlarmDao.java b/BackEnd/src/main/java/com/mcmp/costbe/alarm/dao/AlarmDao.java new file mode 100644 index 0000000..2d70adf --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/alarm/dao/AlarmDao.java @@ -0,0 +1,20 @@ +package com.mcmp.costbe.alarm.dao; + +import com.mcmp.costbe.alarm.model.AlarmHistoryItemModel; +import com.mcmp.costbe.alarm.model.AlarmHistoryReqModel; +import org.mybatis.spring.SqlSessionTemplate; +import org.springframework.stereotype.Repository; + +import javax.annotation.Resource; +import java.util.List; + +@Repository +public class AlarmDao { + + @Resource(name="sqlSessionTemplateBill") + private SqlSessionTemplate sqlSession; + + public List getAlarmHistory(AlarmHistoryReqModel req){ + return sqlSession.selectList("alarm.getAlarmHistory", req); + } +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/alarm/model/AlarmHistoryItemModel.java b/BackEnd/src/main/java/com/mcmp/costbe/alarm/model/AlarmHistoryItemModel.java new file mode 100644 index 0000000..395c179 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/alarm/model/AlarmHistoryItemModel.java @@ -0,0 +1,21 @@ +package com.mcmp.costbe.alarm.model; + +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +public class AlarmHistoryItemModel { + + private String event_type; + private String resource_id; + private String resource_type; + private LocalDateTime occure_time; + private String csp_type; + private String account_id; + private String urgency; + private String plan; + private String note; + private String project_cd; + +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/alarm/model/AlarmHistoryReqModel.java b/BackEnd/src/main/java/com/mcmp/costbe/alarm/model/AlarmHistoryReqModel.java new file mode 100644 index 0000000..4f4ee9f --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/alarm/model/AlarmHistoryReqModel.java @@ -0,0 +1,21 @@ +package com.mcmp.costbe.alarm.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.time.LocalDate; +import java.util.List; + +@Data +public class AlarmHistoryReqModel { + + @Schema(description = "선택 csp", example = "[\"AWS\"]", required = true) + private List selectedCsps; + @Schema(description = "선택 워크스페이스", example = "workspaceCode1", required = false, deprecated = true) + private String selectedWorkspace; + @Schema(description = "선택 프로젝트", example = "[\"projectCode1\", \"projectCode2\"]", required = true) + private List selectedProjects; + + @Schema(required = false) + private LocalDate curDate; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/alarm/model/AlarmHistoryRstModel.java b/BackEnd/src/main/java/com/mcmp/costbe/alarm/model/AlarmHistoryRstModel.java new file mode 100644 index 0000000..6614ed5 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/alarm/model/AlarmHistoryRstModel.java @@ -0,0 +1,13 @@ +package com.mcmp.costbe.alarm.model; + +import lombok.Builder; +import lombok.Data; + +import java.time.LocalDate; +import java.util.List; + +@Data +public class AlarmHistoryRstModel extends AlarmHistoryReqModel{ + + private List alarmHistory; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/alarm/service/AlarmHistoryService.java b/BackEnd/src/main/java/com/mcmp/costbe/alarm/service/AlarmHistoryService.java new file mode 100644 index 0000000..2a2ce14 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/alarm/service/AlarmHistoryService.java @@ -0,0 +1,36 @@ +package com.mcmp.costbe.alarm.service; + +import com.mcmp.costbe.alarm.dao.AlarmDao; +import com.mcmp.costbe.alarm.model.AlarmHistoryReqModel; +import com.mcmp.costbe.alarm.model.AlarmHistoryRstModel; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.time.LocalDate; + +@Service +@Slf4j +public class AlarmHistoryService { + + @Autowired + private AlarmDao alarmDao; + + public AlarmHistoryRstModel getAlarmHistory(AlarmHistoryReqModel req) { + try{ + AlarmHistoryRstModel result = new AlarmHistoryRstModel(); + req.setCurDate(LocalDate.now()); + + result.setCurDate(req.getCurDate()); + result.setAlarmHistory(alarmDao.getAlarmHistory(req)); + result.setSelectedCsps(req.getSelectedCsps()); + result.setSelectedProjects(req.getSelectedProjects()); + + return result; + } catch (Exception e){ + e.printStackTrace(); + throw new RuntimeException(); + } + + } +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/common/model/LocalDateModel.java b/BackEnd/src/main/java/com/mcmp/costbe/common/model/LocalDateModel.java new file mode 100644 index 0000000..9013349 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/common/model/LocalDateModel.java @@ -0,0 +1,10 @@ +package com.mcmp.costbe.common.model; + +import lombok.Data; + +import java.time.LocalDate; + +@Data +public class LocalDateModel { + private LocalDate date; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/common/model/PrevMonthsModel.java b/BackEnd/src/main/java/com/mcmp/costbe/common/model/PrevMonthsModel.java new file mode 100644 index 0000000..b8ebe08 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/common/model/PrevMonthsModel.java @@ -0,0 +1,10 @@ +package com.mcmp.costbe.common.model; + +import lombok.Data; + +import java.util.List; + +@Data +public class PrevMonthsModel { + private List prevMonths; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/common/service/DateCalculator.java b/BackEnd/src/main/java/com/mcmp/costbe/common/service/DateCalculator.java index 7c66236..d37f9cb 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/common/service/DateCalculator.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/common/service/DateCalculator.java @@ -19,7 +19,7 @@ public DateRangeModel dateRangeCalculator(String now) { YearMonth yearMonth = YearMonth.of(date.getYear(), date.getMonth()); LocalDateTime firstDayOfMonth = yearMonth.atDay(1).atStartOfDay(); - LocalDateTime lastDayOfMonth = yearMonth.atEndOfMonth().atTime(23, 59, 59); + LocalDateTime lastDayOfMonth = yearMonth.plusMonths(1).atDay(1).atStartOfDay(); DateRangeModel result = new DateRangeModel(); result.setStartDate(firstDayOfMonth); @@ -43,6 +43,18 @@ public String curMonthDate(String now) { return date.format(formatter); } + + public List getLast12Months(String now){ + LocalDate curDate = LocalDate.parse(now, DateTimeFormatter.ofPattern("yyyyMMdd")); + + List last12Months = new ArrayList<>(); + for(int i=0; i<12; i++){ + String yearMonth = curDate.minusMonths(i).format(DateTimeFormatter.ofPattern("yyyyMM")); + last12Months.add(yearMonth); + } + + return last12Months; + } public List calculatePeriodDates(String Date, String periodType) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd"); @@ -95,4 +107,9 @@ public LocalDate curUTCLocalDate(){ return ZonedDateTime.now(ZoneId.of("UTC")).toLocalDate(); } + public LocalDate curLocalDate(){ + return ZonedDateTime.now().toLocalDate(); + } + + } diff --git a/BackEnd/src/main/java/com/mcmp/costbe/common/service/ExceptionService.java b/BackEnd/src/main/java/com/mcmp/costbe/common/service/ExceptionService.java new file mode 100644 index 0000000..58f0948 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/common/service/ExceptionService.java @@ -0,0 +1,15 @@ +package com.mcmp.costbe.common.service; + +import org.springframework.jdbc.BadSqlGrammarException; +import org.springframework.stereotype.Service; + +@Service +public class ExceptionService { + + public boolean isTableNotFound(BadSqlGrammarException ex){ + String msg = ex.getCause().getMessage(); + + return msg != null && msg.contains("Table") && (msg.contains("doesn't exist") || msg.contains("not found") || + msg.contains("does not exist") || msg.contains("ORA-00942")); + } +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/invoice/BillingInvoiceController.java b/BackEnd/src/main/java/com/mcmp/costbe/invoice/BillingInvoiceController.java index abd7c9f..f46ee63 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/invoice/BillingInvoiceController.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/invoice/BillingInvoiceController.java @@ -38,7 +38,7 @@ public class BillingInvoiceController { public ResponseEntity getBillingBaseInfo(@RequestBody BillingInvoiceBaseInfoReqModel req) throws IOException { ResultModel result = new ResultModel(); try { - List data = billingInvoiceService.getCurMonthBill(req); + List data = billingInvoiceService.getBaseInfo(req); result.setData(data); } catch (Exception e){ e.printStackTrace(); diff --git a/BackEnd/src/main/java/com/mcmp/costbe/invoice/InvoiceController.java b/BackEnd/src/main/java/com/mcmp/costbe/invoice/InvoiceController.java index 9300bce..86c0eef 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/invoice/InvoiceController.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/invoice/InvoiceController.java @@ -31,7 +31,7 @@ public class InvoiceController { private InvoiceService invoiceService; @PostMapping(path = "/getSummary") - @Operation(summary = "빌링 인보이스 날짜별 조회", description = "CSP별 빌링 인보이스 비용을 날짜별로 확인한다.") + @Operation(summary = "빌링 인보이스 월별 조회", description = "CSP별 빌링 인보이스 비용을 월별로 확인한다.") @ApiResponses(value={ @ApiResponse(responseCode = "200", description = "성공", content = {@Content(schema = @Schema(implementation = SummaryResModel.class))}), @@ -40,25 +40,15 @@ public class InvoiceController { public ResponseEntity getSummary(@RequestBody SummaryReqModel req){ ResultModel result = new ResultModel(); try{ - List calPeriodDated = dateCalculator.calculatePeriodDates(req.getToday(), req.getSelectedPeriod()); - + req.setPrevMonths(dateCalculator.getLast12Months(req.getToday())); SummaryResModel resultData = new SummaryResModel(); resultData.setCurYear(req.getToday().substring(0, 4)); resultData.setCurMonth(req.getToday().substring(4, 6)); - resultData.setCurDay(req.getToday().substring(6, 8)); - resultData.setDates(dateCalculator.LocalDateTimeToString(calPeriodDated)); + resultData.setYearMonths(req.getPrevMonths()); resultData.setSelectedCsps(req.getSelectedCsps()); resultData.setSelectedProjects(req.getSelectedProjects()); - resultData.setSelectedPeriod(req.getSelectedPeriod()); - List summaryBillResult = new ArrayList<>(); - summaryBillResult.add(invoiceService.getAWSSummary(req, calPeriodDated)); - summaryBillResult.add(invoiceService.getGCPSummary()); - summaryBillResult.add(invoiceService.getAzureSummary()); - summaryBillResult.add(invoiceService.getNcpSummary()); - SummaryBillItemModel totalSummary = invoiceService.getTotalSummary(summaryBillResult); - summaryBillResult.add(totalSummary); - + List summaryBillResult = invoiceService.getInvoiceSummaryBill(req); resultData.setSummaryBill(summaryBillResult); result.setData(resultData); } catch (Exception e){ diff --git a/BackEnd/src/main/java/com/mcmp/costbe/invoice/dao/InvoiceDao.java b/BackEnd/src/main/java/com/mcmp/costbe/invoice/dao/InvoiceDao.java index 3618ef0..8e8d574 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/invoice/dao/InvoiceDao.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/invoice/dao/InvoiceDao.java @@ -2,6 +2,7 @@ import com.mcmp.costbe.invoice.model.InvoiceItemModel; import com.mcmp.costbe.invoice.model.InvoiceReqModel; +import com.mcmp.costbe.invoice.model.SummaryBillItemModel; import com.mcmp.costbe.invoice.model.SummaryReqModel; import com.mcmp.costbe.usage.model.bill.BillingWidgetModel; import com.mcmp.costbe.usage.model.bill.BillingWidgetReqModel; @@ -16,8 +17,8 @@ public class InvoiceDao { @Resource(name="sqlSessionTemplateBill") private SqlSessionTemplate sqlSession; - public Double getSummary(SummaryReqModel req){ - return sqlSession.selectOne("invoice.getAWSSummaryBill", req); + public List getSummaryBill(SummaryReqModel req){ + return sqlSession.selectList("invoice.getSummaryBill", req); } public List getAWSInvoice(InvoiceReqModel req){ diff --git a/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/BillingInvoiceBaseInfoReqModel.java b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/BillingInvoiceBaseInfoReqModel.java index dde91b6..cc4f9a1 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/BillingInvoiceBaseInfoReqModel.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/BillingInvoiceBaseInfoReqModel.java @@ -1,5 +1,6 @@ package com.mcmp.costbe.invoice.model; +import com.mcmp.costbe.usage.model.bill.YearMonthModel; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; @@ -8,7 +9,7 @@ @Data @Schema(description = "빌링 인보이스 요약 조회 요청 모델") -public class BillingInvoiceBaseInfoReqModel { +public class BillingInvoiceBaseInfoReqModel extends YearMonthModel { @Schema(description = "오늘 날짜", example = "20240620", required = true) private String today; @Schema(required = false) @@ -17,7 +18,7 @@ public class BillingInvoiceBaseInfoReqModel { private LocalDateTime curMonthEndDate; @Schema(description = "CSP", example = "[\"AWS\"]", required = true) private List selectedCsps; - @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = true) + @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = false, deprecated = true) private String selectedWorkspace; @Schema(description = "프로젝트 코드", example = "[\"projectCode1\", \"projectCode2\"]", required = true) private List selectedProjects; diff --git a/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/InvoiceReqModel.java b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/InvoiceReqModel.java index 6365bde..35e80a1 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/InvoiceReqModel.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/InvoiceReqModel.java @@ -1,5 +1,6 @@ package com.mcmp.costbe.invoice.model; +import com.mcmp.costbe.usage.model.bill.YearMonthModel; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; @@ -8,14 +9,14 @@ @Data @Schema(description = "이번달 빌링 인보이스 조회 요청 모델") -public class InvoiceReqModel { +public class InvoiceReqModel extends YearMonthModel { @Schema(description = "오늘 날짜", example = "20240620", required = true) private String today; @Schema(description = "프로젝트 코드", example = "[\"projectCode1\", \"projectCode2\"]", required = true) private List selectedProjects; @Schema(description = "CSP", example = "[\"AWS\"]", required = true) private List selectedCsps; - @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = true) + @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = false, deprecated = true) private String selectedWorkspace; @Schema(required = false) diff --git a/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryBillItemModel.java b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryBillItemModel.java index 88efe26..f026604 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryBillItemModel.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryBillItemModel.java @@ -7,9 +7,11 @@ @Data public class SummaryBillItemModel { - @Schema(description = "CSP", example = "AWS") + private String csp; + @Schema(description = "연도월", example = "202406") + private String yearMonth; @Schema(description = "비용") - private List bill; + private Double bill; } diff --git a/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryBillItemsModel.java b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryBillItemsModel.java new file mode 100644 index 0000000..0b41a82 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryBillItemsModel.java @@ -0,0 +1,20 @@ +package com.mcmp.costbe.invoice.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.List; + +@Data +public class SummaryBillItemsModel { + @Schema(description = "CSP", example = "AWS") + private String csp; + @Schema(description = "해당 CSP에 대한 비용 월별 목록") + private List monthlyBill; + + public SummaryBillItemsModel(String csp, List monthlyBill){ + this.csp = csp; + this.monthlyBill = monthlyBill; + } +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryReqModel.java b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryReqModel.java index 3026b20..e6144fd 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryReqModel.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryReqModel.java @@ -1,5 +1,6 @@ package com.mcmp.costbe.invoice.model; +import com.mcmp.costbe.common.model.PrevMonthsModel; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; import org.apache.tomcat.jni.Local; @@ -9,17 +10,15 @@ @Data @Schema(description = "빌링 인보이스 날짜별 조회 요청 모델") -public class SummaryReqModel { +public class SummaryReqModel extends PrevMonthsModel { @Schema(description = "오늘 날짜", example = "20240620", required = true) private String today; @Schema(description = "프로젝트 코드", example = "[\"projectCode1\", \"projectCode2\"]", required = true) private List selectedProjects; - @Schema(description = "CSP", example = "[\"AWS\"]", required = true) + @Schema(description = "CSP", example = "[\"AWS\"]", required = false) private List selectedCsps; - @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = true) + @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = false, deprecated = true) private String selectedWorkspace; - @Schema(description = "기간 선택 (7days, 3months, 30days)", example = "7days", required = true) - private String selectedPeriod; @Schema(required = false) private List summaryPeriod; diff --git a/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryResModel.java b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryResModel.java index 4db39bb..fcf7e5c 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryResModel.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/invoice/model/SummaryResModel.java @@ -12,14 +12,10 @@ public class SummaryResModel { private String curMonth; @Schema(description = "년도", example = "2024") private String curYear; - @Schema(description = "일", example = "20") - private String curDay; @Schema(description = "csp 별 비용") - private List summaryBill; - @Schema(description = "선택 날짜 목록", example = "[\"2024-06-13\",\"2024-06-14\",\"2024-06-15\",\"2024-06-16\",\"2024-06-17\",\"2024-06-18\",\"2024-06-19\"]") - private List dates; - @Schema(description = "선택 기간", example = "7days") - private String selectedPeriod; + private List summaryBill; + @Schema(description = "지난 12달 목록", example = "[\"202406\",\"202405\",\"202404\", ... ,\"202307\"]") + private List yearMonths; @Schema(description = "선택 프로젝트 코드", example = "[\"projectCode1\", \"projectCode2\"]") private List selectedProjects; @Schema(description = "선택 CSP", example = "[\"AWS\"]") diff --git a/BackEnd/src/main/java/com/mcmp/costbe/invoice/service/BillingInvoiceService.java b/BackEnd/src/main/java/com/mcmp/costbe/invoice/service/BillingInvoiceService.java index 05fc474..602f214 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/invoice/service/BillingInvoiceService.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/invoice/service/BillingInvoiceService.java @@ -2,16 +2,20 @@ import com.mcmp.costbe.common.model.DateRangeModel; import com.mcmp.costbe.common.service.DateCalculator; +import com.mcmp.costbe.common.service.ExceptionService; import com.mcmp.costbe.invoice.dao.BillingInvoiceDao; import com.mcmp.costbe.invoice.model.BillingInvoiceBaseInfoModel; import com.mcmp.costbe.invoice.model.BillingInvoiceBaseInfoReqModel; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.List; @Service +@Slf4j public class BillingInvoiceService { @Autowired @@ -20,38 +24,52 @@ public class BillingInvoiceService { @Autowired private DateCalculator dateCalculator; - public List getCurMonthBill(BillingInvoiceBaseInfoReqModel req){ + @Autowired + private ExceptionService exceptionService; + + public List getBaseInfo(BillingInvoiceBaseInfoReqModel req){ String curMonth = dateCalculator.curMonthDate(req.getToday()); DateRangeModel curMonthRange = dateCalculator.dateRangeCalculator(curMonth); req.setCurMonthStartDate(curMonthRange.getStartDate()); req.setCurMonthEndDate(curMonthRange.getEndDate()); + req.setYear_month(req.getToday().substring(0, 6)); - BillingInvoiceBaseInfoModel aws_result = billingInvoiceDao.getCurMonthBill(req); - aws_result.setCsp("AWS"); - aws_result.setColorClass("bg-google"); + List result = new ArrayList<>(); + try { + BillingInvoiceBaseInfoModel aws_result = billingInvoiceDao.getCurMonthBill(req); + aws_result.setCsp("AWS"); + aws_result.setColorClass("bg-google"); - BillingInvoiceBaseInfoModel gcp_result = new BillingInvoiceBaseInfoModel(); - gcp_result.setCsp("GCP"); - gcp_result.setCost(0); - gcp_result.setColorClass("bg-facebook"); + BillingInvoiceBaseInfoModel gcp_result = new BillingInvoiceBaseInfoModel(); + gcp_result.setCsp("GCP"); + gcp_result.setCost(0); + gcp_result.setColorClass("bg-facebook"); - BillingInvoiceBaseInfoModel azure_result = new BillingInvoiceBaseInfoModel(); - azure_result.setCsp("AZURE"); - azure_result.setCost(0); - azure_result.setColorClass("bg-red"); + BillingInvoiceBaseInfoModel azure_result = new BillingInvoiceBaseInfoModel(); + azure_result.setCsp("AZURE"); + azure_result.setCost(0); + azure_result.setColorClass("bg-red"); - BillingInvoiceBaseInfoModel ncp_result = new BillingInvoiceBaseInfoModel(); - ncp_result.setCsp("NCP"); - ncp_result.setCost(0); - ncp_result.setColorClass("bg-green"); + BillingInvoiceBaseInfoModel ncp_result = new BillingInvoiceBaseInfoModel(); + ncp_result.setCsp("NCP"); + ncp_result.setCost(0); + ncp_result.setColorClass("bg-green"); - List result = new ArrayList<>(); - result.add(aws_result); - result.add(gcp_result); - result.add(azure_result); - result.add(ncp_result); + result.add(aws_result); + result.add(gcp_result); + result.add(azure_result); + result.add(ncp_result); + + } catch (BadSqlGrammarException ex){ + if(exceptionService.isTableNotFound(ex)){ + log.warn("[BaseInfo Widget Log] NotFoundTable : {}", ex.getMessage()); + } else { + ex.printStackTrace(); + throw new RuntimeException(); + } + } return result; } } diff --git a/BackEnd/src/main/java/com/mcmp/costbe/invoice/service/InvoiceService.java b/BackEnd/src/main/java/com/mcmp/costbe/invoice/service/InvoiceService.java index 15e5255..f0cd565 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/invoice/service/InvoiceService.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/invoice/service/InvoiceService.java @@ -2,15 +2,19 @@ import com.mcmp.costbe.common.model.DateRangeModel; import com.mcmp.costbe.common.service.DateCalculator; +import com.mcmp.costbe.common.service.ExceptionService; import com.mcmp.costbe.invoice.dao.InvoiceDao; import com.mcmp.costbe.invoice.model.*; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.stereotype.Service; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; @Service @Slf4j @@ -22,73 +26,19 @@ public class InvoiceService { @Autowired private InvoiceDao invoiceDao; - public SummaryBillItemModel getAWSSummary(SummaryReqModel req, List calPeriodDated ){ - - - req.setSummaryPeriod(calPeriodDated); - - List bill = new ArrayList<>(); - for(int i = 0; i bill = List.of(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0); - - SummaryBillItemModel summaryGCPItem = new SummaryBillItemModel(); - summaryGCPItem.setBill(bill); - summaryGCPItem.setCsp("GCP"); - - - return summaryGCPItem; - } - - public SummaryBillItemModel getAzureSummary(){ - List bill = List.of(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0); - - SummaryBillItemModel summaryAzureItem = new SummaryBillItemModel(); - summaryAzureItem.setBill(bill); - summaryAzureItem.setCsp("AZURE"); - - - return summaryAzureItem; - } - - public SummaryBillItemModel getNcpSummary(){ - List bill = List.of(0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0); - - SummaryBillItemModel summaryNcpItem = new SummaryBillItemModel(); - summaryNcpItem.setBill(bill); - summaryNcpItem.setCsp("NCP"); - - - return summaryNcpItem; - } - - public SummaryBillItemModel getTotalSummary(List summaryBill){ - SummaryBillItemModel summaryTotalItem = new SummaryBillItemModel(); - List totalBill = new ArrayList<>(); + @Autowired + private ExceptionService exceptionService; - for(int i=0; i getInvoiceSummaryBill(SummaryReqModel req){ + List summary = invoiceDao.getSummaryBill(req); + Map> groupedByCsp = summary.stream() + .collect(Collectors.groupingBy(SummaryBillItemModel::getCsp)); - summaryTotalItem.setBill(totalBill); - summaryTotalItem.setCsp("Total"); + List result = groupedByCsp.entrySet().stream() + .map(item -> new SummaryBillItemsModel(item.getKey(), item.getValue())) + .collect(Collectors.toList()); - return summaryTotalItem; + return result; } public List getAWSInvoice(InvoiceReqModel req){ @@ -96,6 +46,19 @@ public List getAWSInvoice(InvoiceReqModel req){ req.setCurMonthStartDate(dateRange.getStartDate()); req.setCurMonthEndDate(dateRange.getEndDate()); - return invoiceDao.getAWSInvoice(req); + req.setYear_month(req.getToday().substring(0, 6)); + + try { + return invoiceDao.getAWSInvoice(req); + } catch (BadSqlGrammarException ex){ + if(exceptionService.isTableNotFound(ex)){ + log.warn("[Invoice Widget Log] NotFoundTable : {}", ex.getMessage()); + } else { + ex.printStackTrace(); + throw new RuntimeException(); + } + } + + return null; } } diff --git a/BackEnd/src/main/java/com/mcmp/costbe/opti/OptiController.java b/BackEnd/src/main/java/com/mcmp/costbe/opti/OptiController.java index 1082826..81bb4dc 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/opti/OptiController.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/opti/OptiController.java @@ -3,10 +3,7 @@ import com.mcmp.costbe.common.model.ResultModel; import com.mcmp.costbe.invoice.model.BillingInvoiceBaseInfoModel; import com.mcmp.costbe.invoice.model.BillingInvoiceBaseInfoReqModel; -import com.mcmp.costbe.opti.model.UnusedQueryParamModel; -import com.mcmp.costbe.opti.model.UnusedQueryRstModel; -import com.mcmp.costbe.opti.model.UnusedReqModel; -import com.mcmp.costbe.opti.model.UnusedRstModel; +import com.mcmp.costbe.opti.model.*; import com.mcmp.costbe.opti.service.OptiService; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.ArraySchema; @@ -33,14 +30,14 @@ public class OptiController { @Autowired private OptiService optiService; - @PostMapping(path = "/unusedRec") - @Operation(summary = "미사용 자원(추천) 조회", description = "이번달 CSP별 요약된 빌링 인보이스를 확인한다.") + @PostMapping(path = "/unusedRcmd") + @Operation(summary = "미사용 자원(추천) 조회", description = "최근 24시간동안 과금이 발생한 리소스에 대하여 미사용 자원을 추천한다.") @ApiResponses(value={ @ApiResponse(responseCode = "200", description = "성공", - content = {@Content(array = @ArraySchema(schema = @Schema(implementation = UnusedQueryRstModel.class)))}), + content = {@Content(schema = @Schema(implementation = UnusedQueryRstModel.class))}), @ApiResponse(responseCode = "500", description = "서버 오류", content = {@Content(examples = {})}) }) - public ResponseEntity getUnusedRec(@RequestBody UnusedReqModel req){ + public ResponseEntity getUnusedRcmd(@RequestBody UnusedReqModel req){ ResultModel result = new ResultModel(); try { UnusedRstModel data = optiService.getOptiUnused(req); @@ -52,4 +49,42 @@ public ResponseEntity getUnusedRec(@RequestBody UnusedReqModel req){ return ResponseEntity.ok(result); } + @PostMapping(path = "/abnormalRcmd") + @Operation(summary = "이상 비용 조회", description = "최근 24시간동안 과금이 발생한 서비스들의 이상 비용 여부를 확인한다.") + @ApiResponses(value={ + @ApiResponse(responseCode = "200", description = "성공", + content = {@Content(schema = @Schema(implementation = AbnormalRstModel.class))}), + @ApiResponse(responseCode = "500", description = "서버 오류", content = {@Content(examples = {})}) + }) + public ResponseEntity getAbrnormalRcmd(@RequestBody AbnormalReqModel req){ + ResultModel result = new ResultModel(); + try { + AbnormalRstModel data = optiService.getOptiAbrnomal(req); + result.setData(data); + } catch (Exception e){ + e.printStackTrace(); + result.setError(500, "Failed to get Abnormal Service Info"); + } + return ResponseEntity.ok(result); + } + + @PostMapping(path = "/instOptiSizeRcmd") + @Operation(summary = "인스턴스 사이즈 추천 조회", description = "사용중인 인스턴스의 추천 사이즈를 확인한다.") + @ApiResponses(value={ + @ApiResponse(responseCode = "200", description = "성공", + content = {@Content(schema = @Schema(implementation = InstOptiSizeRstModel.class))}), + @ApiResponse(responseCode = "500", description = "서버 오류", content = {@Content(examples = {})}) + }) + public ResponseEntity getInstOptiSizeRcmd(@RequestBody InstOptiSizeReqModel req){ + ResultModel result = new ResultModel(); + try { + InstOptiSizeRstModel data = optiService.getInstOptiSizeRcmd(req); + result.setData(data); + } catch (Exception e){ + e.printStackTrace(); + result.setError(500, "Failed to get OptiSize Recommend Service Info"); + } + return ResponseEntity.ok(result); + } + } diff --git a/BackEnd/src/main/java/com/mcmp/costbe/opti/dao/OptiDao.java b/BackEnd/src/main/java/com/mcmp/costbe/opti/dao/OptiDao.java index 65f6cd1..29a991d 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/opti/dao/OptiDao.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/opti/dao/OptiDao.java @@ -1,7 +1,6 @@ package com.mcmp.costbe.opti.dao; -import com.mcmp.costbe.opti.model.UnusedQueryParamModel; -import com.mcmp.costbe.opti.model.UnusedQueryRstModel; +import com.mcmp.costbe.opti.model.*; import org.mybatis.spring.SqlSessionTemplate; import org.springframework.stereotype.Repository; @@ -18,4 +17,12 @@ public List getOptiUnused(UnusedQueryParamModel param){ return sqlSession.selectList("opti.getOptiUnused", param); } + public List getOptiAbnormal(AbnormalReqModel param){ + return sqlSession.selectList("opti.getOptiAbnormal", param); + } + + public List getInstOptiSize(InstOptiSizeReqModel param){ + return sqlSession.selectList("opti.getInstOptiSize", param); + } + } diff --git a/BackEnd/src/main/java/com/mcmp/costbe/opti/model/AbnoramlItemModel.java b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/AbnoramlItemModel.java new file mode 100644 index 0000000..8bdabab --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/AbnoramlItemModel.java @@ -0,0 +1,26 @@ +package com.mcmp.costbe.opti.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +public class AbnoramlItemModel { + @Schema(description = "서비스 코드", example = "AmazonEC2") + private String product_cd; + + @Schema(description = "이상비용 위험도", example = "Alarm") + private String abnormal_rating; + + @Schema(description = "지난달 사용 차이률", example = "200%") + private double percentage_point; + + @Schema(description = "오늘 과금 비용(USD)", example = "20.0") + private double standard_cost; + + @Schema(description = "지난달 사용 과금 비용 평균(USD)", example = "10.0") + private double subject_cost; + + @Schema(description = "csp 종류", example = "AWS") + private String csp_type; + +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/opti/model/AbnormalReqModel.java b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/AbnormalReqModel.java new file mode 100644 index 0000000..cfc8db8 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/AbnormalReqModel.java @@ -0,0 +1,23 @@ +package com.mcmp.costbe.opti.model; + +import com.mcmp.costbe.common.model.LocalDateModel; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "이상 비용 조회 요청 모델") +public class AbnormalReqModel extends LocalDateModel { + @Schema(description = "오늘 날짜", example = "20240608", required = true) + private String today; + + @Schema(description = "선택 워크스페이스", example = "workspaceCode1", required = false, deprecated = true) + private String selectedWorkspace; + + @Schema(description = "선택 프로젝트", example = "[\"projectCode1\", \"projectCode2\"]", required = true) + private List selectedProjects; + + @Schema(description = "선택 csp", example = "[\"AWS\"]", required = false) + private List selectedCsps; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/opti/model/AbnormalRstModel.java b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/AbnormalRstModel.java new file mode 100644 index 0000000..006e1ae --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/AbnormalRstModel.java @@ -0,0 +1,26 @@ +package com.mcmp.costbe.opti.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Builder; +import lombok.Data; + +import java.time.LocalDate; +import java.util.List; + +@Data +@Builder +public class AbnormalRstModel { + + @Schema(description = "오늘 날짜", example = "20240608") + private String today; + + @Schema(description = "이상비용 목록") + private List abnoramlItems; + + + @Schema(description = "선택 프로젝트", example = "[\"projectCode1\", \"projectCode2\"]") + private List selectedProjects; + + @Schema(description = "선택 csp", example = "[\"AWS\"]") + private List selectedCsps; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/opti/model/InstOptiSizeItemModel.java b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/InstOptiSizeItemModel.java new file mode 100644 index 0000000..acfb207 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/InstOptiSizeItemModel.java @@ -0,0 +1,24 @@ +package com.mcmp.costbe.opti.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +@Data +public class InstOptiSizeItemModel { + @Schema(description = "리소스 ID", example = "i-02eeeeeeee1234") + private String resource_id; + @Schema(description = "csp 종류", example = "AWS") + private String csp_type; + @Schema(description = "계정", example = "111111111111") + private String account; + @Schema(description = "기존타입", example = "t2.small") + private String origin_type; + @Schema(description = "추천타입", example = "t2.micro") + private String rcmd_type; + @Schema(description = "추천 방향", example = "Down") + private String plan_type; + @Schema(description = "기존타입 비용", example = "0.0288") + private double origin_usd; + @Schema(description = "추천타입 비용", example = "0.0144") + private double rcmd_usd; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/opti/model/InstOptiSizeReqModel.java b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/InstOptiSizeReqModel.java new file mode 100644 index 0000000..c25a0e1 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/InstOptiSizeReqModel.java @@ -0,0 +1,23 @@ +package com.mcmp.costbe.opti.model; + +import com.mcmp.costbe.common.model.LocalDateModel; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Data; + +import java.util.List; + +@Data +@Schema(description = "인스턴스 사이즈 추천 조회") +public class InstOptiSizeReqModel extends LocalDateModel { + @Schema(description = "오늘 날짜", example = "20240608", required = true) + private String today; + + @Schema(description = "선택 워크스페이스", example = "workspaceCode1", required = false, deprecated = true) + private String selectedWorkspace; + + @Schema(description = "선택 프로젝트", example = "[\"projectCode1\", \"projectCode2\"]", required = true) + private List selectedProjects; + + @Schema(description = "선택 csp", example = "[\"AWS\"]", required = false) + private List selectedCsps; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/opti/model/InstOptiSizeRstModel.java b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/InstOptiSizeRstModel.java new file mode 100644 index 0000000..570a731 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/InstOptiSizeRstModel.java @@ -0,0 +1,24 @@ +package com.mcmp.costbe.opti.model; + +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +@Data +@Builder +public class InstOptiSizeRstModel { + @Schema(description = "오늘 날짜", example = "20240608") + private String today; + + @Schema(description = "인스턴스 추천 사이즈 목록") + private List optiSizeItems; + + + @Schema(description = "선택 프로젝트", example = "[\"projectCode1\", \"projectCode2\"]") + private List selectedProjects; + + @Schema(description = "선택 csp", example = "[\"AWS\"]") + private List selectedCsps; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/opti/model/UnusedQueryParamModel.java b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/UnusedQueryParamModel.java index a55ed05..3ac5080 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/opti/model/UnusedQueryParamModel.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/UnusedQueryParamModel.java @@ -7,4 +7,5 @@ @Data public class UnusedQueryParamModel extends UnusedReqModel { private LocalDate curDate; + private String lastYearMonth; } diff --git a/BackEnd/src/main/java/com/mcmp/costbe/opti/model/UnusedReqModel.java b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/UnusedReqModel.java index 915c78a..c4b2497 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/opti/model/UnusedReqModel.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/opti/model/UnusedReqModel.java @@ -9,7 +9,7 @@ public class UnusedReqModel { @Schema(description = "선택 csp", example = "[\"AWS\"]", required = true) private List selectedCsps; - @Schema(description = "선택 워크스페이스", example = "workspaceCode1") + @Schema(description = "선택 워크스페이스", example = "workspaceCode1", required = false, deprecated = true) private String selectedWorkspace; @Schema(description = "선택 프로젝트", example = "[\"projectCode1\", \"projectCode2\"]", required = true) private List selectedProjects; diff --git a/BackEnd/src/main/java/com/mcmp/costbe/opti/service/OptiService.java b/BackEnd/src/main/java/com/mcmp/costbe/opti/service/OptiService.java index 3218e7f..c6f012e 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/opti/service/OptiService.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/opti/service/OptiService.java @@ -2,15 +2,13 @@ import com.mcmp.costbe.common.service.DateCalculator; import com.mcmp.costbe.opti.dao.OptiDao; -import com.mcmp.costbe.opti.model.UnusedQueryParamModel; -import com.mcmp.costbe.opti.model.UnusedQueryRstModel; -import com.mcmp.costbe.opti.model.UnusedReqModel; -import com.mcmp.costbe.opti.model.UnusedRstModel; +import com.mcmp.costbe.opti.model.*; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.List; @Service @@ -25,19 +23,19 @@ public class OptiService { public UnusedRstModel getOptiUnused(UnusedReqModel req){ try { UnusedRstModel rst = new UnusedRstModel(); - LocalDate curDate = dateCalculator.curUTCLocalDate(); + LocalDate curDate = dateCalculator.curLocalDate(); + LocalDate lastYM = curDate.minusMonths(1); UnusedQueryParamModel queryParam = new UnusedQueryParamModel(); queryParam.setCurDate(curDate); + queryParam.setLastYearMonth(lastYM.format(DateTimeFormatter.ofPattern("yyyyMM"))); queryParam.setSelectedCsps(req.getSelectedCsps()); queryParam.setSelectedProjects(req.getSelectedProjects()); - queryParam.setSelectedWorkspace(req.getSelectedWorkspace()); List queryRst = optiDao.getOptiUnused(queryParam); rst.setCurDate(curDate); rst.setSelectedCsps(req.getSelectedCsps()); - rst.setSelectedWorkspace(req.getSelectedWorkspace()); rst.setSelectedProjects(req.getSelectedProjects()); rst.setUnusedRec(queryRst); @@ -48,4 +46,44 @@ public UnusedRstModel getOptiUnused(UnusedReqModel req){ throw new RuntimeException(); } } + + public AbnormalRstModel getOptiAbrnomal(AbnormalReqModel req){ + try{ + LocalDate date = LocalDate.now(); + req.setDate(date); + + AbnormalRstModel result = AbnormalRstModel.builder() + .today(date.toString()) + .abnoramlItems(optiDao.getOptiAbnormal(req)) + .selectedProjects(req.getSelectedProjects()) + .selectedCsps(req.getSelectedCsps()) + .build(); + + return result; + + } catch (Exception e){ + e.printStackTrace(); + throw new RuntimeException(); + } + } + + public InstOptiSizeRstModel getInstOptiSizeRcmd(InstOptiSizeReqModel req){ + try{ + LocalDate date = LocalDate.now(); + req.setDate(date); + + InstOptiSizeRstModel result = InstOptiSizeRstModel.builder() + .today(date.toString()) + .optiSizeItems(optiDao.getInstOptiSize(req)) + .selectedProjects(req.getSelectedProjects()) + .selectedCsps(req.getSelectedCsps()) + .build(); + + return result; + + } catch (Exception e){ + e.printStackTrace(); + throw new RuntimeException(); + } + } } diff --git a/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/TumblebugController.java b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/TumblebugController.java new file mode 100644 index 0000000..d75b253 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/TumblebugController.java @@ -0,0 +1,30 @@ +package com.mcmp.costbe.tumblebugMeta; + +import com.mcmp.costbe.common.model.ResultModel; +import com.mcmp.costbe.tumblebugMeta.service.VMMetaService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + + +@RestController +@RequestMapping(path = "/api/v2") +public class TumblebugController { + + @Autowired + private VMMetaService vmMetaService; + + @GetMapping(path = "/updateRscMeta") + public ResponseEntity updateTBBRscMeta(){ + ResultModel result = new ResultModel(); + try{ + vmMetaService.getTBBResourceMetaInfo(); + } catch (Exception e){ + e.printStackTrace(); + result.setError(500, "Fail to Update (Tumblebug) ResourceMeta"); + } + return ResponseEntity.ok(result); + } +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/dao/TBBDao.java b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/dao/TBBDao.java new file mode 100644 index 0000000..1e90608 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/dao/TBBDao.java @@ -0,0 +1,21 @@ +package com.mcmp.costbe.tumblebugMeta.dao; + +import com.mcmp.costbe.tumblebugMeta.model.ResourcegroupMetaModel; +import org.mybatis.spring.SqlSessionTemplate; +import org.springframework.stereotype.Repository; + +import javax.annotation.Resource; +import java.util.List; + +@Repository +public class TBBDao { + + @Resource(name="sqlSessionTemplateBill") + private SqlSessionTemplate sqlSession; + + public void insertTBBServicegroupMeta(List rscMetas){ + sqlSession.insert("tbb.insertTBBServicegroupMeta", rscMetas); + } + + +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/ResourcegroupMetaModel.java b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/ResourcegroupMetaModel.java new file mode 100644 index 0000000..dc8a5bd --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/ResourcegroupMetaModel.java @@ -0,0 +1,22 @@ +package com.mcmp.costbe.tumblebugMeta.model; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class ResourcegroupMetaModel { + private String cspType; + private String cspAccount; + private String cspInstanceid; + private String serviceCd; + private String serviceNm; + private String serviceUid; + private String vmId; + private String vmUid; + private String vmNm; + private String mciId; + private String mciUid; + private String mciNm; + private String instanceRunningStatus; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/mci/TBBMCIItemModel.java b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/mci/TBBMCIItemModel.java new file mode 100644 index 0000000..611d0c6 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/mci/TBBMCIItemModel.java @@ -0,0 +1,29 @@ +package com.mcmp.costbe.tumblebugMeta.model.mci; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class TBBMCIItemModel { + private String configureCloudAdaptiveNetwork; + private String description; + private String id; + private String installMonAgent; + private Map label; + private String name; + private List newVmList; + private String placementAlgo; + private String resourceType; + private String status; + private Object statusCount; + private String systemLabel; + private String systemMessage; + private String targetAction; + private String targetStatus; + private String uid; + private List vm; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/mci/TBBMCIModel.java b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/mci/TBBMCIModel.java new file mode 100644 index 0000000..a6be090 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/mci/TBBMCIModel.java @@ -0,0 +1,10 @@ +package com.mcmp.costbe.tumblebugMeta.model.mci; + +import lombok.Data; + +import java.util.List; + +@Data +public class TBBMCIModel { + private List mci; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/mci/TbVmInfoModel.java b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/mci/TbVmInfoModel.java new file mode 100644 index 0000000..3d6c63c --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/mci/TbVmInfoModel.java @@ -0,0 +1,51 @@ +package com.mcmp.costbe.tumblebugMeta.model.mci; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class TbVmInfoModel { + private Map addtionalDetails; + private Object connectionConfig; + private String connectionName; + private String createdTime; + private String cspImageName; + private String cspResourceId; + private String cspResourceName; + private String cspSpecName; + private String cspSshKeyId; + private String cspSubnetId; + private String cspVNetId; + private List dataDiskIds; + private String description; + private String id; + private String imageId; + private Map label; + private Object location; + private String monAgentStatus; + private String name; + private String networkAgentStatus; + private String networkInterface; + private String privateDNS; + private String privateIP; + private String publicDNS; + private String publicIP; + private Object region; + private String resourceType; + private String specId; + private String sshKeyId; + private String status; + private String subGroupId; + private String subnetId; + private String targetAction; + private String targetStatus; + private String uid; + private String vNetId; + private String vmUserName; + private String vmUserPassword; + +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/ns/TBBNSItemModel.java b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/ns/TBBNSItemModel.java new file mode 100644 index 0000000..29dcdf5 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/ns/TBBNSItemModel.java @@ -0,0 +1,12 @@ +package com.mcmp.costbe.tumblebugMeta.model.ns; + +import lombok.Data; + +@Data +public class TBBNSItemModel { + + private String id; + private String name; + private String resourceType; + private String uid; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/ns/TBBNSModel.java b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/ns/TBBNSModel.java new file mode 100644 index 0000000..37fa090 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/model/ns/TBBNSModel.java @@ -0,0 +1,13 @@ +package com.mcmp.costbe.tumblebugMeta.model.ns; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +import java.util.List; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class TBBNSModel { + private List ns; + private List output; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/service/VMMetaService.java b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/service/VMMetaService.java new file mode 100644 index 0000000..5a56d48 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/tumblebugMeta/service/VMMetaService.java @@ -0,0 +1,175 @@ +package com.mcmp.costbe.tumblebugMeta.service; + +import com.mcmp.costbe.tumblebugMeta.dao.TBBDao; +import com.mcmp.costbe.tumblebugMeta.model.ResourcegroupMetaModel; +import com.mcmp.costbe.tumblebugMeta.model.mci.TBBMCIItemModel; +import com.mcmp.costbe.tumblebugMeta.model.mci.TBBMCIModel; +import com.mcmp.costbe.tumblebugMeta.model.mci.TbVmInfoModel; +import com.mcmp.costbe.tumblebugMeta.model.ns.TBBNSItemModel; +import com.mcmp.costbe.tumblebugMeta.model.ns.TBBNSModel; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.*; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.RestTemplate; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; + +@Service +@Slf4j +public class VMMetaService { + + @Value("${tumblebug.url}") + public String tumblebugUrl; + + @Value("${tumblebug.username}") + public String tumblebugUserNM; + + @Value("${tumblebug.password}") + public String tumblebugPW; + + @Autowired + private TBBDao tbbDao; + + + public List getTbbNS(){ + + String apiUrl = String.format("%s/ns", tumblebugUrl); + RestTemplate restTemplate = new RestTemplate(); + + String auth = tumblebugUserNM + ":" + tumblebugPW; + byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.UTF_8)); + String authHeader = "Basic " + new String(encodedAuth); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.set("Authorization", authHeader); + HttpEntity httpEntity = new HttpEntity<>(httpHeaders); + + try{ + ResponseEntity responseEntity = restTemplate.exchange(apiUrl, HttpMethod.GET, httpEntity, TBBNSModel.class); + TBBNSModel response = responseEntity.getBody(); + + if(response.getNs() != null && !response.getNs().isEmpty()){ + return response.getNs(); + }else{ + log.warn("TUMBLEBUG META - NS IS EMPTY => response : {}", response); + return new ArrayList<>(); + } + } catch (HttpClientErrorException | HttpServerErrorException clientError) { + HttpStatus statusCode = clientError.getStatusCode(); + log.error("FAIL TO GET TUMBLEBUG META - NS : " + statusCode); + throw new RuntimeException(); + } catch (Exception e){ + log.error("FAIL TO GET TUMBLEBUG META - NS : " +e.getMessage()); + throw new RuntimeException(); + } + } + + public List getTBBMCI(TBBNSItemModel item){ + + if(item != null){ + String apiUrl = String.format("%s/ns/%s/mci", tumblebugUrl, item.getId()); + RestTemplate restTemplate = new RestTemplate(); + + String auth = tumblebugUserNM + ":" + tumblebugPW; + byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.UTF_8)); + String authHeader = "Basic " + new String(encodedAuth); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.set("Authorization", authHeader); + HttpEntity httpEntity = new HttpEntity<>(httpHeaders); + + try{ + ResponseEntity responseEntity = restTemplate.exchange(apiUrl, HttpMethod.GET, httpEntity, TBBMCIModel.class); + TBBMCIModel response = responseEntity.getBody(); + + if(response.getMci() != null && !response.getMci().isEmpty()){ + return response.getMci(); + }else{ + log.warn("TUMBLEBUG META - MCI => MCI IS EMPTY => response : {}", response); + return new ArrayList<>(); + } + } catch (HttpClientErrorException | HttpServerErrorException clientError) { + HttpStatus statusCode = clientError.getStatusCode(); + log.error("FAIL TO GET TUMBLEBUG META - MCI => NS ID : {}, error code : {}", item.getId(), statusCode); + throw new RuntimeException(); + } catch (Exception e){ + log.error("FAIL TO GET TUMBLEBUG META - MCI => NS ID : {}", item.getId()); + throw new RuntimeException(); + } + + } else { + log.error("[ERROR] : GET TUMBLEBUG META - MCI => NS IS EMPTY"); + return new ArrayList<>(); + } + } + + public void getTBBResourceMetaInfo() throws InterruptedException { + List nsList = getTbbNS(); + + for(TBBNSItemModel ns : nsList){ + + Thread.sleep(2000); + List mciList = getTBBMCI(ns); + + for(TBBMCIItemModel mci : mciList){ + if("mci".equals(mci.getResourceType())){ + try{ + List resourcegroupMetaList = new ArrayList<>(); + + List vmList = mci.getVm(); + for(TbVmInfoModel vm : vmList){ + String vmStatus; + if(!vm.getStatus().isEmpty()){ + vmStatus = switch (vm.getStatus()){ + case "Running" -> "Y"; + case "Failed" -> "N"; + default -> "Y"; + }; + }else { + vmStatus = "Y"; + } + + if(vm.getCspResourceId() != null){ + ResourcegroupMetaModel vmInfo = ResourcegroupMetaModel.builder() + .cspType("AWS") + .cspAccount("mcmpcostopti") + .cspInstanceid(vm.getCspResourceId()) + .serviceCd(ns.getId()) + .serviceNm(ns.getName()) + .serviceUid(ns.getUid()) + .vmId(vm.getId()) + .vmUid(vm.getUid()) + .vmNm(vm.getName()) + .mciId(mci.getId()) + .mciUid(mci.getUid()) + .mciNm(mci.getName()) + .instanceRunningStatus(vmStatus) + .build(); + + resourcegroupMetaList.add(vmInfo); + } + + } + + if(resourcegroupMetaList.size() >= 1){ + tbbDao.insertTBBServicegroupMeta(resourcegroupMetaList); + } + + } catch (Exception e){ + e.printStackTrace(); + throw new RuntimeException(); + } + + } + } + } + } + +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/usage/UsageController.java b/BackEnd/src/main/java/com/mcmp/costbe/usage/UsageController.java index 70bc5a3..5e52395 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/usage/UsageController.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/usage/UsageController.java @@ -25,9 +25,10 @@ public class UsageController { @Autowired private UsageService usageService; + @Deprecated @GetMapping(path = "/getWorkspaces") @Tag(name = "User", description = "User API") - @Operation(summary = "워크스페이스 목록 조회", description = "워크스페이스 목록을 조회합니다.") + @Operation(summary = "워크스페이스 목록 조회", description = "워크스페이스 목록을 조회합니다.", deprecated = true) @ApiResponses(value={ @ApiResponse(responseCode = "200", description = "성공"), @ApiResponse(responseCode = "500", description = "서버 오류", content = {@Content(examples = {})}) @@ -44,9 +45,10 @@ public ResponseEntity getWorkspaces(){ return ResponseEntity.ok(result); } + @Deprecated @GetMapping(path = "/getProjects") @Tag(name = "User", description = "User API") - @Operation(summary = "프로젝트 목록 조회", description = "워크스페이스에 속한 프로젝트 목록을 조회합니다.") + @Operation(summary = "프로젝트 목록 조회", description = "워크스페이스에 속한 프로젝트 목록을 조회합니다.", deprecated = true) @ApiResponses(value={ @ApiResponse(responseCode = "200", description = "성공"), @ApiResponse(responseCode = "500", description = "서버 오류", content = {@Content(examples = {})}) @@ -76,7 +78,7 @@ public ResponseEntity getProjects( public ResponseEntity getCurMonthBill(@RequestBody BillingWidgetReqModel req) throws IOException { ResultModel result = new ResultModel(); try{ - BillingWidgetModel data = usageService.getBillingWidget(req); + BillingWidgetModel data = usageService.getBillingMonthlyWidget(req); result.setData(data); } catch (Exception e){ e.printStackTrace(); @@ -107,7 +109,7 @@ public ResponseEntity getTop5Bill(@RequestBody Top5WidgetReqModel req) throws IO @PostMapping(path = "/getBillAsset") @Tag(name = "Cost Dashboard", description = "Cost Dashboard overview API") - @Operation(summary = "이번달 리소스 사용량 및 비용 조회", description = "이번달 사용한 리소스의 unit과 비용을 확인합니다.") + @Operation(summary = "이번달 사용 서비스별 비용 조회", description = "이번달 사용한 서비스(VM, DB 등) 단위의 비용을 확인합니다.") @ApiResponses(value={ @ApiResponse(responseCode = "200", description = "성공", content = {@Content(schema = @Schema(implementation = BillingAssetWidgetModel.class))}), diff --git a/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/BillingAssetReqModel.java b/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/BillingAssetReqModel.java index 1242737..76151b9 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/BillingAssetReqModel.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/BillingAssetReqModel.java @@ -8,14 +8,14 @@ @Data @Schema(description = "이번달 리소스 사용량 및 비용 조회 요청 모델") -public class BillingAssetReqModel { +public class BillingAssetReqModel extends YearMonthModel { @Schema(description = "오늘 날짜", example = "20240620", required = true) private String today; @Schema(description = "프로젝트 코드", example = "[\"projectCode1\", \"projectCode2\"]", required = true) private List selectedProjects; @Schema(description = "CSP", example = "[\"AWS\"]", required = true) private List selectedCsps; - @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = true) + @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = false, deprecated = true) private String selectedWorkspace; @Schema(required = false) private LocalDateTime curMonthStartDate; diff --git a/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/BillingWidgetReqModel.java b/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/BillingWidgetReqModel.java index 3e2bebd..01bd724 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/BillingWidgetReqModel.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/BillingWidgetReqModel.java @@ -1,5 +1,6 @@ package com.mcmp.costbe.usage.model.bill; +import com.mcmp.costbe.common.model.PrevMonthsModel; import io.swagger.v3.oas.annotations.media.Schema; import lombok.Data; @@ -8,7 +9,7 @@ @Data @Schema(description = "이번달 비용 조회 요청 모델") -public class BillingWidgetReqModel { +public class BillingWidgetReqModel extends PrevMonthsModel { @Schema(description = "오늘 날짜", example = "20240620", required = true) private String today; @@ -19,7 +20,7 @@ public class BillingWidgetReqModel { @Schema(description = "CSP", example = "[\"AWS\"]", required = true) private List selectedCsps; - @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = true) + @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = false, deprecated = true) private String selectedWorkspace; @Schema(required = false) @@ -30,4 +31,10 @@ public class BillingWidgetReqModel { private LocalDateTime prevMonthStartDate; @Schema(required = false) private LocalDateTime prevMonthEndDate; + + @Schema(required = false) + private String curYearMonth; + + @Schema(required = false) + private String prevYearMonth; } diff --git a/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/Top5WidgetReqModel.java b/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/Top5WidgetReqModel.java index 725ee17..3ef364e 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/Top5WidgetReqModel.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/Top5WidgetReqModel.java @@ -8,14 +8,14 @@ @Data @Schema(description = "이번달 상위 5개 리소스 비용 조회 요청 모델") -public class Top5WidgetReqModel { +public class Top5WidgetReqModel extends YearMonthModel { @Schema(description = "오늘 날짜", example = "20240620", required = true) private String today; @Schema(description = "프로젝트 코드", example = "[\"projectCode1\", \"projectCode2\"]", required = true) private List selectedProjects; @Schema(description = "CSP", example = "[\"AWS\"]", required = true) private List selectedCsps; - @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = true) + @Schema(description = "워크스페이스 코드", example = "workspaceCode1", required = false, deprecated = true) private String selectedWorkspace; @Schema(required = false) diff --git a/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/YearMonthModel.java b/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/YearMonthModel.java new file mode 100644 index 0000000..6cdd014 --- /dev/null +++ b/BackEnd/src/main/java/com/mcmp/costbe/usage/model/bill/YearMonthModel.java @@ -0,0 +1,8 @@ +package com.mcmp.costbe.usage.model.bill; + +import lombok.Data; + +@Data +public class YearMonthModel { + private String year_month; +} diff --git a/BackEnd/src/main/java/com/mcmp/costbe/usage/service/UsageService.java b/BackEnd/src/main/java/com/mcmp/costbe/usage/service/UsageService.java index 5668637..2a93c92 100644 --- a/BackEnd/src/main/java/com/mcmp/costbe/usage/service/UsageService.java +++ b/BackEnd/src/main/java/com/mcmp/costbe/usage/service/UsageService.java @@ -2,6 +2,7 @@ import com.mcmp.costbe.common.model.DateRangeModel; import com.mcmp.costbe.common.service.DateCalculator; +import com.mcmp.costbe.common.service.ExceptionService; import com.mcmp.costbe.resourceMapping.aws.AWSResourceMapping; import com.mcmp.costbe.usage.dao.BillDao; import com.mcmp.costbe.usage.dao.FilterDao; @@ -11,6 +12,7 @@ import io.swagger.v3.oas.annotations.tags.Tag; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -26,6 +28,9 @@ public class UsageService { @Autowired private BillDao billDao; + @Autowired + private ExceptionService exceptionService; + @Autowired private DateCalculator dateCalculator; @@ -37,15 +42,11 @@ public List getProjects(String workspaceCD){ return filterDao.getProjects(workspaceCD); } - public BillingWidgetModel getBillingWidget(BillingWidgetReqModel req){ + public BillingWidgetModel getBillingMonthlyWidget(BillingWidgetReqModel req){ String prevMonth = dateCalculator.prevMonthdate(req.getToday()); - DateRangeModel curMonthRange = dateCalculator.dateRangeCalculator(req.getToday()); - DateRangeModel prevMonthRange = dateCalculator.dateRangeCalculator(prevMonth); - - req.setCurMonthStartDate(curMonthRange.getStartDate()); - req.setCurMonthEndDate(curMonthRange.getEndDate()); - req.setPrevMonthStartDate(prevMonthRange.getStartDate()); - req.setPrevMonthEndDate(prevMonthRange.getEndDate()); + req.setCurYearMonth(req.getToday().substring(0, 6)); + req.setPrevYearMonth(prevMonth.substring(0, 6)); + req.setPrevMonths(dateCalculator.getLast12Months(req.getToday())); BillingWidgetModel result = billDao.getCurPrevMonthBill(req); result.setCurYear(req.getToday().substring(0, 4)); @@ -65,34 +66,44 @@ public BillingWidgetModel getBillingWidget(BillingWidgetReqModel req){ return result; } - public Top5WidgetModel getTop5Bill(Top5WidgetReqModel req){ + public Top5WidgetModel getTop5Bill(Top5WidgetReqModel req) { DateRangeModel curMonthRange = dateCalculator.dateRangeCalculator(req.getToday()); req.setCurMonthStartDate(curMonthRange.getStartDate()); req.setCurMonthEndDate(curMonthRange.getEndDate()); + req.setYear_month(req.getToday().substring(0,6)); Top5WidgetModel result = new Top5WidgetModel(); - List top5bill = billDao.getTop5Bill(req); - - for(Top5BillModel item : top5bill){ - if("others".equals(item.getResourceNm())){ - item.setIsOthers(true); - }else{ - item.setIsOthers(false); - } - } + try{ + List top5bill = billDao.getTop5Bill(req); + + for(Top5BillModel item : top5bill){ + if("others".equals(item.getResourceNm())){ + item.setIsOthers(true); + }else{ + item.setIsOthers(false); + } + } + + if(top5bill.isEmpty()){ + Top5BillModel temp = new Top5BillModel(); + temp.setBill(0.0); + temp.setCsp("Null"); + temp.setIsOthers(false); + temp.setResourceNm("Null"); - System.out.println(top5bill.size()); - if(top5bill.isEmpty()){ - Top5BillModel temp = new Top5BillModel(); - temp.setBill(0.0); - temp.setCsp("Null"); - temp.setIsOthers(false); - temp.setResourceNm("Null"); + top5bill.add(temp); + } - top5bill.add(temp); + result.setTop5bill(top5bill); + } catch (BadSqlGrammarException ex){ + if(exceptionService.isTableNotFound(ex)){ + log.warn("[Top5 Widget Log]NotFoundTable : {}", ex.getMessage()); + }else { + ex.printStackTrace(); + throw new RuntimeException(); + } } - result.setTop5bill(top5bill); result.setSelectedProjects(req.getSelectedProjects()); result.setSelectedCsps(req.getSelectedCsps()); result.setCurYear(req.getToday().substring(0, 4)); @@ -107,29 +118,39 @@ public BillingAssetWidgetModel getBillAsset(BillingAssetReqModel req){ DateRangeModel curMonthRange = dateCalculator.dateRangeCalculator(req.getToday()); req.setCurMonthStartDate(curMonthRange.getStartDate()); req.setCurMonthEndDate(curMonthRange.getEndDate()); + req.setYear_month(req.getToday().substring(0, 6)); List familyCode = List.of("Virtual Machine", "Storage", "Database", "LB"); List billingAsset = new ArrayList<>(); - for(String item : familyCode){ - BillingAssetModel familyItem = new BillingAssetModel(); - List childProducts = AWSResourceMapping.getData(item); - req.setAWSChildProducts(childProducts); - - List childItem = billDao.getBillAssetChild(req); - double childsTotalBill = 0.0; - Integer childsTotalUnit = 0; - for(BillingAssetChildModel child : childItem){ - childsTotalUnit += child.getUnit(); - childsTotalBill += child.getBill(); + try{ + for(String item : familyCode){ + BillingAssetModel familyItem = new BillingAssetModel(); + List childProducts = AWSResourceMapping.getData(item); + req.setAWSChildProducts(childProducts); + + List childItem = billDao.getBillAssetChild(req); + double childsTotalBill = 0.0; + Integer childsTotalUnit = 0; + for(BillingAssetChildModel child : childItem){ + childsTotalUnit += child.getUnit(); + childsTotalBill += child.getBill(); + } + + familyItem.setChildProductCode(childItem); + familyItem.setTotalCost(childsTotalBill); + familyItem.setTotalUnit(childsTotalUnit); + familyItem.setFamilyProductCode(item); + + billingAsset.add(familyItem); + } + } catch (BadSqlGrammarException ex){ + if(exceptionService.isTableNotFound(ex)){ + log.warn("[BillAsset Widget Log]NotFoundTable : {}", ex.getMessage()); + } else { + ex.printStackTrace(); + throw new RuntimeException(); } - - familyItem.setChildProductCode(childItem); - familyItem.setTotalCost(childsTotalBill); - familyItem.setTotalUnit(childsTotalUnit); - familyItem.setFamilyProductCode(item); - - billingAsset.add(familyItem); } result.setSelectedProjects(req.getSelectedProjects()); diff --git a/BackEnd/src/main/resources/application.properties b/BackEnd/src/main/resources/application.properties index 7cc492b..ad52c4f 100644 --- a/BackEnd/src/main/resources/application.properties +++ b/BackEnd/src/main/resources/application.properties @@ -27,3 +27,7 @@ springdoc.swagger-ui.tags-sorter=alpha springdoc.swagger-ui.operations-sorter=alpha springdoc.default-consumes-media-type=application/json;charset=UTF-8 springdoc.default-produces-media-type=application/json;charset=UTF-8 + +tumblebug.url=https://local.tumblebug.com/tumblebug +tumblebug.username=default +tumblebug.password=default diff --git a/BackEnd/src/main/resources/mapper/bill/alarm_SQL.xml b/BackEnd/src/main/resources/mapper/bill/alarm_SQL.xml new file mode 100644 index 0000000..2fed44a --- /dev/null +++ b/BackEnd/src/main/resources/mapper/bill/alarm_SQL.xml @@ -0,0 +1,38 @@ + + + + + + diff --git a/BackEnd/src/main/resources/mapper/bill/bill_SQL.xml b/BackEnd/src/main/resources/mapper/bill/bill_SQL.xml index f62ba30..58aebdd 100644 --- a/BackEnd/src/main/resources/mapper/bill/bill_SQL.xml +++ b/BackEnd/src/main/resources/mapper/bill/bill_SQL.xml @@ -21,96 +21,68 @@ - SELECT COALESCE(sum(co.lineitem_unblendedcost),0) - FROM cur_origin co - LEFT JOIN servicegroup_meta sm - ON co.lineitem_resourceid = sm.csp_instanceid - WHERE 1=1 - - AND sm.csp_type IN - - #{selectedCsp} + + + + + diff --git a/BackEnd/src/main/resources/mapper/bill/tbb_SQL.xml b/BackEnd/src/main/resources/mapper/bill/tbb_SQL.xml new file mode 100644 index 0000000..463fa0e --- /dev/null +++ b/BackEnd/src/main/resources/mapper/bill/tbb_SQL.xml @@ -0,0 +1,28 @@ + + + + + INSERT INTO servicegroup_meta(csp_type, csp_account, csp_instanceid, service_cd, service_nm, service_uid + , vm_id, vm_uid, vm_nm, mci_id, mci_uid, mci_nm, instance_running_status) + VALUES + + (#{item.cspType} + , #{item.cspAccount} + , #{item.cspInstanceid} + , #{item.serviceCd} + , #{item.serviceNm} + , #{item.serviceUid} + , #{item.vmId} + , #{item.vmUid} + , #{item.vmNm} + , #{item.mciId} + , #{item.mciUid} + , #{item.mciNm} + , #{item.instanceRunningStatus}) + + ON DUPLICATE KEY UPDATE service_nm = values(service_nm) + , vm_nm = values(vm_nm) + , mci_nm = values(mci_nm) + , instance_running_status = values(instance_running_status) + + diff --git a/BackEnd/src/test/java/com/mcmp/costbe/dateRangeTest.java b/BackEnd/src/test/java/com/mcmp/costbe/dateRangeTest.java index 3c9dd13..85a2648 100644 --- a/BackEnd/src/test/java/com/mcmp/costbe/dateRangeTest.java +++ b/BackEnd/src/test/java/com/mcmp/costbe/dateRangeTest.java @@ -25,10 +25,8 @@ public class dateRangeTest { @Test public void testCal(){ - List result = dateCalculator.calculatePeriodDates("20240321", "30days"); +// List result = dateCalculator.calculatePeriodDates("20240321", "30days"); // System.out.println(result); -// List result = AWSResourceMapping.getData("Database"); - System.out.println(result); } } diff --git a/assetCollector/.gitignore b/assetCollector/.gitignore new file mode 100644 index 0000000..549e00a --- /dev/null +++ b/assetCollector/.gitignore @@ -0,0 +1,33 @@ +HELP.md +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ diff --git a/assetCollector/.mvn/wrapper/maven-wrapper.properties b/assetCollector/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..d58dfb7 --- /dev/null +++ b/assetCollector/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +wrapperVersion=3.3.2 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.9/apache-maven-3.9.9-bin.zip diff --git a/assetCollector/Dockerfile b/assetCollector/Dockerfile new file mode 100644 index 0000000..e2ab61e --- /dev/null +++ b/assetCollector/Dockerfile @@ -0,0 +1,13 @@ +# Build stage +FROM maven:3.8.5-openjdk-17-slim AS build +WORKDIR /app +COPY pom.xml . +COPY src ./src +RUN mvn clean package -DskipTests + +# Run stage +FROM openjdk:17-jdk-slim +WORKDIR /app +COPY --from=build /app/target/*.jar app.jar +EXPOSE 8091 +ENTRYPOINT ["java","-jar","app.jar"] diff --git a/assetCollector/mvnw b/assetCollector/mvnw new file mode 100644 index 0000000..19529dd --- /dev/null +++ b/assetCollector/mvnw @@ -0,0 +1,259 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.2 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/assetCollector/mvnw.cmd b/assetCollector/mvnw.cmd new file mode 100644 index 0000000..249bdf3 --- /dev/null +++ b/assetCollector/mvnw.cmd @@ -0,0 +1,149 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.2 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" (%__MVNW_CMD__% %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' +$MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain" +if ($env:MAVEN_USER_HOME) { + $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain" +} +$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/assetCollector/pom.xml b/assetCollector/pom.xml new file mode 100644 index 0000000..740455d --- /dev/null +++ b/assetCollector/pom.xml @@ -0,0 +1,116 @@ + + + 4.0.0 + + org.springframework.boot + spring-boot-starter-parent + 3.2.10 + + + com.mcmp + assetCollector + 0.0.1-SNAPSHOT + assetCollector + assetCollector + + + + + + + + + + + + + + + 17 + + + + org.springframework.boot + spring-boot-starter-web + + + + org.springframework.boot + spring-boot-devtools + runtime + true + + + org.projectlombok + lombok + true + + + org.springframework.boot + spring-boot-starter-test + test + + + org.mybatis + mybatis-spring + 3.0.0 + + + org.mybatis + mybatis + 3.5.11 + + + org.springframework + spring-jdbc + 6.1.5 + + + org.mariadb.jdbc + mariadb-java-client + 3.1.4 + runtime + + + org.springframework.boot + spring-boot-starter-batch + + + org.springframework.boot + spring-boot-starter-quartz + + + org.springframework.batch + spring-batch-integration + + + org.springframework.batch + spring-batch-test + test + + + com.integralblue + log4jdbc-spring-boot-starter + 2.0.0 + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + + + + + + diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/AssetCollectorApplication.java b/assetCollector/src/main/java/com/mcmp/assetcollector/AssetCollectorApplication.java new file mode 100644 index 0000000..5263015 --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/AssetCollectorApplication.java @@ -0,0 +1,13 @@ +package com.mcmp.assetcollector; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class AssetCollectorApplication { + + public static void main(String[] args) { + SpringApplication.run(AssetCollectorApplication.class, args); + } + +} diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/batch/AssetCollect.java b/assetCollector/src/main/java/com/mcmp/assetcollector/batch/AssetCollect.java new file mode 100644 index 0000000..8f800af --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/batch/AssetCollect.java @@ -0,0 +1,149 @@ +package com.mcmp.assetcollector.batch; + + +import com.mcmp.assetcollector.model.batch.RSRCAssetComputeMetricModel; +import com.mcmp.assetcollector.model.batch.RunningInstanceModel; +import com.mcmp.assetcollector.model.service.RSRCAssetUsageItemModel; +import com.mcmp.assetcollector.model.service.RSRCAssetUsageModel; +import com.mcmp.assetcollector.service.AssetCollectService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.ibatis.session.SqlSessionFactory; +import org.mybatis.spring.batch.MyBatisBatchItemWriter; +import org.mybatis.spring.batch.MyBatisPagingItemReader; +import org.mybatis.spring.batch.builder.MyBatisBatchItemWriterBuilder; +import org.mybatis.spring.batch.builder.MyBatisPagingItemReaderBuilder; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobScope; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.core.step.skip.AlwaysSkipItemSkipPolicy; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemWriter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.BadSqlGrammarException; +import org.springframework.transaction.PlatformTransactionManager; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.List; +import java.util.PrimitiveIterator; + +@Slf4j +@Configuration +@RequiredArgsConstructor +public class AssetCollect { + + @Autowired + @Qualifier("sqlSessionBatch") + private SqlSessionFactory sqlSessionFactory; + + @Autowired + private AssetCollectService assetCollectService; + + @Bean + public SkipLogListener skipLogListener(){ + return new SkipLogListener(); + } + + @Bean + public Job assetCollectJob(JobRepository jobRepository, Step assetCollectStep) { + return new JobBuilder("assetCollectJob", jobRepository) + .start(assetCollectStep) + .build(); + } + + @Bean + @JobScope + public Step assetCollectStep(JobRepository jobRepository, PlatformTransactionManager platformTransactionManager){ + return new StepBuilder("assetCollectStep", jobRepository) + .>chunk(1, platformTransactionManager) + .reader(reader()) + .processor(processor()) + .writer(writerListItems(writerItems())) + .startLimit(2) + .faultTolerant() + .retry(Exception.class) + .retryLimit(2) + .skip(Exception.class) + .skipPolicy(new AlwaysSkipItemSkipPolicy()) + .listener(skipLogListener()) + .build(); + } + + @Bean + @StepScope + public MyBatisPagingItemReader reader() { + return new MyBatisPagingItemReaderBuilder() + .sqlSessionFactory(sqlSessionFactory) + .queryId("asset.getAssetRunningInstance") + .pageSize(1) + .build(); + } + + @StepScope + public ItemProcessor> processor() { + return item -> { + + Thread.sleep(5000); + List data = new ArrayList<>(); + try{ + List tempData = assetCollectService.getRSRCCpuUsageHistory(item.getNsID(), item.getMciID(), item.getVmID()); + + if(!tempData.isEmpty()){ + for(RSRCAssetUsageItemModel tempDataItem : tempData){ + try { + Double amount = Double.parseDouble(tempDataItem.getValue()); + + RSRCAssetComputeMetricModel dataItem = RSRCAssetComputeMetricModel.builder() + .cspType(item.getCspType()) + .cspAccount(item.getCspAccount()) + .cspInstanceid(item.getInstanceID()) + .collectDt(Timestamp.from(tempDataItem.getTimestamp())) + .metricType("cpu") + .metricAmount(amount) + .resourceStatus("running") + .resourceSpotYn("N") + .build(); + + data.add(dataItem); + } catch (NumberFormatException e){ + log.warn("Invalid number format for NS ID: {}, MIC ID: {}, VM ID: {}, item: {}", item.getNsID(), item.getMciID(), item.getVmID(), item); + } + } + } else { + return null; + } + + }catch (Exception ex){ + ex.printStackTrace(); + return null; + } + return data; + }; + } + + @Bean + public ItemWriter> writerListItems(MyBatisBatchItemWriter writerItems){ + return chunks -> { + for(List chunk : chunks){ + writerItems.write(new Chunk<>(chunk)); + } + }; + } + + @Bean + public MyBatisBatchItemWriter writerItems() { + return new MyBatisBatchItemWriterBuilder() + .sqlSessionFactory(sqlSessionFactory) + .statementId("asset.insertRSRCComputeMetric") + .build(); + } +} diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/batch/SkipLogListener.java b/assetCollector/src/main/java/com/mcmp/assetcollector/batch/SkipLogListener.java new file mode 100644 index 0000000..984f300 --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/batch/SkipLogListener.java @@ -0,0 +1,22 @@ +package com.mcmp.assetcollector.batch; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.SkipListener; + +@Slf4j +public class SkipLogListener implements SkipListener { + @Override + public void onSkipInRead(Throwable t) { + log.warn("Skipped during read due to: {}", t.getMessage(), t); + } + + @Override + public void onSkipInWrite(Object item, Throwable t) { + log.warn("Skipped during write for item {} due to: {}", item, t.getMessage(), t); + } + + @Override + public void onSkipInProcess(Object item, Throwable t) { + log.warn("Skipped during process for item {} due to: {}", item, t.getMessage(), t); + } +} diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/config/DatabaseConfig.java b/assetCollector/src/main/java/com/mcmp/assetcollector/config/DatabaseConfig.java new file mode 100644 index 0000000..6791949 --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/config/DatabaseConfig.java @@ -0,0 +1,57 @@ +package com.mcmp.assetcollector.config; + +import com.zaxxer.hikari.HikariDataSource; +import org.apache.ibatis.session.ExecutorType; +import org.apache.ibatis.session.SqlSessionFactory; +import org.mybatis.spring.SqlSessionFactoryBean; +import org.mybatis.spring.SqlSessionTemplate; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.core.io.support.ResourcePatternResolver; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.transaction.PlatformTransactionManager; + +import javax.sql.DataSource; + +@Configuration +public class DatabaseConfig { + @Primary + @Bean(name = "dataSource") + @ConfigurationProperties(prefix = "spring.datasource.hikari.batch") + public DataSource dataSourceBatch() { + return DataSourceBuilder.create().type(HikariDataSource.class).build(); + } + + @Primary + @Bean(name = "transactionManager") + public PlatformTransactionManager transactionManagerBatch() { + return new DataSourceTransactionManager(dataSourceBatch()); + } + + @Primary + @Bean(name = "sqlSessionBatch") + public SqlSessionFactory sqlSessionBatch() throws Exception { + SqlSessionFactoryBean sqlSession = new SqlSessionFactoryBean(); + sqlSession.setDataSource(dataSourceBatch()); + sqlSession.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(ResourcePatternResolver.CLASSPATH_URL_PREFIX + "/mapper/batch/*_SQL.xml")); + return sqlSession.getObject(); + } + + @Bean(name = "sqlSessionSimple") + public SqlSessionFactory sqlSessionSimple() throws Exception { + SqlSessionFactoryBean sqlSession = new SqlSessionFactoryBean(); + sqlSession.setDataSource(dataSourceBatch()); + sqlSession.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(ResourcePatternResolver.CLASSPATH_URL_PREFIX + "/mapper/simple/*_SQL.xml")); + return sqlSession.getObject(); + } + + @Primary + @Bean(name = "sqlSessionTemplateSimple") + public SqlSessionTemplate sqlSessionTemplateSimple() throws Exception { + return new SqlSessionTemplate(sqlSessionSimple(), ExecutorType.SIMPLE); + } +} diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/model/batch/RSRCAssetComputeMetricModel.java b/assetCollector/src/main/java/com/mcmp/assetcollector/model/batch/RSRCAssetComputeMetricModel.java new file mode 100644 index 0000000..e5ee026 --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/model/batch/RSRCAssetComputeMetricModel.java @@ -0,0 +1,19 @@ +package com.mcmp.assetcollector.model.batch; + +import lombok.Builder; +import lombok.Data; + +import java.sql.Timestamp; + +@Data +@Builder +public class RSRCAssetComputeMetricModel { + private String cspType; + private String cspAccount; + private String cspInstanceid; + private Timestamp collectDt; + private String metricType; + private double metricAmount; + private String resourceStatus; + private String resourceSpotYn; +} diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/model/batch/RunningInstanceModel.java b/assetCollector/src/main/java/com/mcmp/assetcollector/model/batch/RunningInstanceModel.java new file mode 100644 index 0000000..5a2c0ce --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/model/batch/RunningInstanceModel.java @@ -0,0 +1,17 @@ +package com.mcmp.assetcollector.model.batch; + +import lombok.Data; + +@Data +public class RunningInstanceModel { + private String cspType; + private String cspAccount; + private String instanceID; + private String nsID; + private String nsUID; + private String vmID; + private String vmUID; + private String mciID; + private String mciUID; + private String instanceRunningStatus; +} diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/model/service/RSRCAssetUsageDataModel.java b/assetCollector/src/main/java/com/mcmp/assetcollector/model/service/RSRCAssetUsageDataModel.java new file mode 100644 index 0000000..038554e --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/model/service/RSRCAssetUsageDataModel.java @@ -0,0 +1,14 @@ +package com.mcmp.assetcollector.model.service; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +import java.util.List; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class RSRCAssetUsageDataModel { + private String metricName; + private String metricUnit; + private List timestampValues; +} diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/model/service/RSRCAssetUsageItemModel.java b/assetCollector/src/main/java/com/mcmp/assetcollector/model/service/RSRCAssetUsageItemModel.java new file mode 100644 index 0000000..e486f30 --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/model/service/RSRCAssetUsageItemModel.java @@ -0,0 +1,13 @@ +package com.mcmp.assetcollector.model.service; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +import java.time.Instant; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class RSRCAssetUsageItemModel { + private Instant timestamp; + private String value; +} diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/model/service/RSRCAssetUsageModel.java b/assetCollector/src/main/java/com/mcmp/assetcollector/model/service/RSRCAssetUsageModel.java new file mode 100644 index 0000000..e4a1112 --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/model/service/RSRCAssetUsageModel.java @@ -0,0 +1,13 @@ +package com.mcmp.assetcollector.model.service; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Data; + +@Data +@JsonIgnoreProperties(ignoreUnknown = true) +public class RSRCAssetUsageModel { + private RSRCAssetUsageDataModel data; + private String rsCode; + private String rsMsg; + private String errorMessage; +} diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/schedule/AssetCltJob.java b/assetCollector/src/main/java/com/mcmp/assetcollector/schedule/AssetCltJob.java new file mode 100644 index 0000000..0021e5d --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/schedule/AssetCltJob.java @@ -0,0 +1,43 @@ +package com.mcmp.assetcollector.schedule; + +import lombok.extern.slf4j.Slf4j; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.scheduling.quartz.QuartzJobBean; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class AssetCltJob extends QuartzJobBean { + + @Autowired + private JobLauncher jobLauncher; + + @Qualifier("assetCollectJob") + @Autowired + private Job assetCollectJob; + + @Override + protected void executeInternal(JobExecutionContext context) throws JobExecutionException { + log.info("Starting AssetCollect Quartz Job"); + + try { + JobParameters jobParameter = new JobParametersBuilder() + .addLong("createTime", System.currentTimeMillis()) + .toJobParameters(); + + jobLauncher.run(assetCollectJob, jobParameter); + } catch (Exception e) { + log.error("Error executing Spring Batch AssetCollect Job", e); + throw new JobExecutionException("Failed to execute Spring AssetCollect Batch Job", e); + } + + log.info("AssetCollect Quartz Job completed"); + } +} diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/schedule/ScheduleConfig.java b/assetCollector/src/main/java/com/mcmp/assetcollector/schedule/ScheduleConfig.java new file mode 100644 index 0000000..799ea0e --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/schedule/ScheduleConfig.java @@ -0,0 +1,34 @@ +package com.mcmp.assetcollector.schedule; + +import org.quartz.*; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class ScheduleConfig { + + @Value("${assetCollectBatchCronSchedule}") + private String assetSchedule; + + @Bean + public JobDetail assetJobDetail() { + return JobBuilder.newJob().ofType(AssetCltJob.class) + .storeDurably() + .withIdentity("assetCollectJob", "assetCollect") + .withDescription("Execute Spring Batch AssetCollect Batch Job with Quartz") + .build(); + } + + @Bean + public Trigger assetJobTrigger(JobDetail assetJobDetail) { + CronScheduleBuilder cronSchedule = CronScheduleBuilder.cronSchedule(assetSchedule); + + return TriggerBuilder.newTrigger().forJob(assetJobDetail) + .withIdentity("assetCollectJobTrigger1", "assetCollect") + .withDescription("AssetCollect Job Trigger") + .withSchedule(cronSchedule) + .build(); + } + +} diff --git a/assetCollector/src/main/java/com/mcmp/assetcollector/service/AssetCollectService.java b/assetCollector/src/main/java/com/mcmp/assetcollector/service/AssetCollectService.java new file mode 100644 index 0000000..7ad0080 --- /dev/null +++ b/assetCollector/src/main/java/com/mcmp/assetcollector/service/AssetCollectService.java @@ -0,0 +1,55 @@ +package com.mcmp.assetcollector.service; + +import com.mcmp.assetcollector.model.service.RSRCAssetUsageDataModel; +import com.mcmp.assetcollector.model.service.RSRCAssetUsageItemModel; +import com.mcmp.assetcollector.model.service.RSRCAssetUsageModel; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.*; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.RestTemplate; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; + +@Service +@Slf4j +public class AssetCollectService { + + @Value("${asset.collect.url}") + private String assetCollectUrl; + + public List getRSRCCpuUsageHistory(String nsID, String mciID, String vmID){ + String apiUrl = String.format("%s/api/o11y/monitoring/%s/%s/target/%s/csp/cpu_usage", assetCollectUrl, nsID, mciID, vmID); + RestTemplate restTemplate = new RestTemplate(); + + HttpHeaders httpHeaders = new HttpHeaders(); + HttpEntity httpEntity = new HttpEntity<>(httpHeaders); + + try{ + ResponseEntity responseEntity = restTemplate.exchange(apiUrl, HttpMethod.GET, httpEntity, RSRCAssetUsageModel.class); + RSRCAssetUsageModel response = responseEntity.getBody(); + RSRCAssetUsageDataModel data = response.getData(); + + if(data == null || data.getTimestampValues() == null || data.getTimestampValues().isEmpty()){ + log.warn("RSRC Asset Usage - {} => NS ID : {}, MIC ID : {}, VM ID : {} => response : {}", (data == null ? "DATA IS EMPTY" : "TimestampValues IS EMPTY"), nsID, mciID, vmID, response); + return new ArrayList<>(); + } + return data.getTimestampValues(); + + } catch (HttpClientErrorException | HttpServerErrorException clientError) { + HttpStatusCode statusCode = clientError.getStatusCode(); + log.error("FAIL TO GET RSRC Asset Usage - NS ID : {}, MIC ID : {}, VM ID : {} => {}", nsID, mciID, vmID, statusCode); + return new ArrayList<>(); + } catch (Exception e){ + log.error("FAIL TO GET RSRC Asset Usage - NS ID : {}, MIC ID : {}, VM ID : {}", nsID, mciID, vmID); + e.printStackTrace(); + return new ArrayList<>(); + } + + } +} diff --git a/assetCollector/src/main/resources/application.properties b/assetCollector/src/main/resources/application.properties new file mode 100644 index 0000000..f610443 --- /dev/null +++ b/assetCollector/src/main/resources/application.properties @@ -0,0 +1,22 @@ +spring.application.name=assetCollector +server.port=8091 + +spring.batch.job.enabled=false + +spring.datasource.hikari.batch.driver-class-name=org.mariadb.jdbc.Driver +spring.datasource.hikari.batch.jdbc-url=jdbc:mariadb://localhost:3306/cost?autoReconnect=true&allowMultiQueries=true&useSSL=false&rewriteBatchedStatements=true +spring.datasource.hikari.batch.username=mcmpcostopti +spring.datasource.hikari.batch.password=0000 +spring.datasource.hikari.batch.initial-size=10 +spring.datasource.hikari.batch.max-total=10 +spring.datasource.hikari.batch.max-idle=10 +spring.datasource.hikari.batch.max-wait-millis=30000 +spring.datasource.hikari.batch.remove-abandoned-on-borrow=true +spring.datasource.hikari.batch.remove-abandoned-timeout=30 + +logging.level.org.springframework.batch=DEBUG +logging.level.org.mybatis=DEBUG + +asset.collect.url=http://monitoring.server:8080 + +assetCollectBatchCronSchedule=0 0 0 * * ? diff --git a/assetCollector/src/main/resources/logback-spring.xml b/assetCollector/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..8abbfc6 --- /dev/null +++ b/assetCollector/src/main/resources/logback-spring.xml @@ -0,0 +1,24 @@ + + + + + UTF-8 + %d{yyyy-MM-dd HH:mm:ss.SSS} %magenta([%thread]) %highlight(%5level) %cyan(%logger) - %msg%n + + + + + + + + + + + + + + + + + + diff --git a/assetCollector/src/main/resources/mapper/batch/asset_SQL.xml b/assetCollector/src/main/resources/mapper/batch/asset_SQL.xml new file mode 100644 index 0000000..e9683e4 --- /dev/null +++ b/assetCollector/src/main/resources/mapper/batch/asset_SQL.xml @@ -0,0 +1,34 @@ + + + + + + + INSERT INTO asset_compute_metric(csp_type, csp_account, csp_instanceid, collect_dt, metric_type, metric_amount, resource_status, resource_spot_yn) + VALUES (#{cspType} + , #{cspAccount} + , #{cspInstanceid} + , #{collectDt} + , #{metricType} + , #{metricAmount} + , #{resourceStatus} + , #{resourceSpotYn}) + ON DUPLICATE KEY UPDATE metric_amount = values(metric_amount) + , resource_status = values(resource_status) + , resource_spot_yn = values(resource_spot_yn) + + diff --git a/assetCollector/src/main/resources/mapper/simple/simple_SQL.xml b/assetCollector/src/main/resources/mapper/simple/simple_SQL.xml new file mode 100644 index 0000000..f9fc451 --- /dev/null +++ b/assetCollector/src/main/resources/mapper/simple/simple_SQL.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/assetCollector/src/test/java/com/mcmp/assetcollector/AssetCollectorApplicationTests.java b/assetCollector/src/test/java/com/mcmp/assetcollector/AssetCollectorApplicationTests.java new file mode 100644 index 0000000..f9b40ce --- /dev/null +++ b/assetCollector/src/test/java/com/mcmp/assetcollector/AssetCollectorApplicationTests.java @@ -0,0 +1,13 @@ +package com.mcmp.assetcollector; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; + +@SpringBootTest +class AssetCollectorApplicationTests { + + @Test + void contextLoads() { + } + +} diff --git a/cost-fe/package.json b/cost-fe/package.json index 019b60f..5094182 100644 --- a/cost-fe/package.json +++ b/cost-fe/package.json @@ -8,11 +8,13 @@ "lint": "vue-cli-service lint" }, "dependencies": { - "apexcharts": "^3.49.1", + "apexcharts": "^3.53.0", "axios": "^1.7.2", "bootstrap": "^5.3.3", "core-js": "^3.8.3", "pinia": "^2.1.7", + "primeicons": "^7.0.0", + "primevue": "^3.29.2", "sass": "^1.77.7", "sass-loader": "^14.2.1", "tabulator-tables": "^6.2.1", diff --git a/cost-fe/public/index.html b/cost-fe/public/index.html index 0e501b9..0e883f3 100644 --- a/cost-fe/public/index.html +++ b/cost-fe/public/index.html @@ -1,5 +1,6 @@ + diff --git a/cost-fe/src/App.vue b/cost-fe/src/App.vue index 9e29c40..e027f80 100644 --- a/cost-fe/src/App.vue +++ b/cost-fe/src/App.vue @@ -1,13 +1,51 @@ diff --git a/cost-fe/src/api/Endpoints.js b/cost-fe/src/api/Endpoints.js index b776061..f2213ec 100644 --- a/cost-fe/src/api/Endpoints.js +++ b/cost-fe/src/api/Endpoints.js @@ -1,12 +1,15 @@ const MainDomain = location.hostname console.log('도메인 확인' + MainDomain) -// const isNumericAndDotsOnly = /^[0-9.]+$/.test(MainDomain); +const isNumericAndDotsOnly = /^[0-9.]+$/.test(MainDomain); let API_BE_URL = ''; let API_ALARM_URL = ''; if(MainDomain.includes('localhost')){ API_BE_URL = 'http://' + MainDomain + ':9090'; - API_ALARM_URL = 'https://' + MainDomain + ':9000'; + API_ALARM_URL = 'http://' + MainDomain + ':9000'; +} else if(isNumericAndDotsOnly) { + API_BE_URL = 'http://' + MainDomain + ':9090'; + API_ALARM_URL = 'http://' + MainDomain + ':9000'; } else { API_BE_URL = 'https://' + MainDomain; API_ALARM_URL = 'https://' + MainDomain; diff --git a/cost-fe/src/components/pages/billinginvoice/BillingInvoiceLayout.vue b/cost-fe/src/components/pages/billinginvoice/BillingInvoiceLayout.vue index 72025ca..93a14d8 100644 --- a/cost-fe/src/components/pages/billinginvoice/BillingInvoiceLayout.vue +++ b/cost-fe/src/components/pages/billinginvoice/BillingInvoiceLayout.vue @@ -7,7 +7,7 @@
- +
@@ -20,7 +20,7 @@ import DashboardSelectbox from '../dashboard/dashboard-selectbox/DashboardSelectbox.vue' import DashboardHeader from '../dashboard/dashboard-header/DashboardHeader.vue' import BaseInfo from './base-info/BaseInfo.vue' -import BillingSummary from './billing-summary/BillingSummary.vue' +import MonthlyOverview from './billing-summary/monthlyOverview.vue' import InvoiceTable from './invoice-table/InvoiceTable.vue' export default { @@ -29,7 +29,7 @@ export default { DashboardHeader, DashboardSelectbox, BaseInfo, - BillingSummary, + MonthlyOverview, InvoiceTable }, data() { @@ -40,7 +40,6 @@ export default { diff --git a/cost-fe/src/components/pages/billinginvoice/billing-summary/BillingSummary.vue b/cost-fe/src/components/pages/billinginvoice/billing-summary/BillingSummary.vue deleted file mode 100644 index 12a2fbe..0000000 --- a/cost-fe/src/components/pages/billinginvoice/billing-summary/BillingSummary.vue +++ /dev/null @@ -1,181 +0,0 @@ - - - - - diff --git a/cost-fe/src/components/pages/billinginvoice/billing-summary/monthlyOverview.vue b/cost-fe/src/components/pages/billinginvoice/billing-summary/monthlyOverview.vue new file mode 100644 index 0000000..dd4a103 --- /dev/null +++ b/cost-fe/src/components/pages/billinginvoice/billing-summary/monthlyOverview.vue @@ -0,0 +1,188 @@ + + + + + diff --git a/cost-fe/src/components/pages/billinginvoice/invoice-table/InvoiceTable.vue b/cost-fe/src/components/pages/billinginvoice/invoice-table/InvoiceTable.vue index 11439d0..06511d1 100644 --- a/cost-fe/src/components/pages/billinginvoice/invoice-table/InvoiceTable.vue +++ b/cost-fe/src/components/pages/billinginvoice/invoice-table/InvoiceTable.vue @@ -4,28 +4,9 @@

Invoices

- -
-
@@ -220,19 +185,9 @@ export default { name: 'DashboardHeader', data() { - return { - avatarUrl: require('@/assets/images/avatars/000m.jpg'), - }; + return {}; }, methods: { - enableDarkMode() { - document.body.classList.add('theme-dark'); - document.body.classList.remove('theme-light'); - }, - enableLightMode() { - document.body.classList.add('theme-light'); - document.body.classList.remove('theme-dark'); - }, isActive(path) { const route = this.$route; return route.path === path; @@ -241,7 +196,4 @@ export default { } - - + diff --git a/cost-fe/src/components/pages/dashboard/dashboard-selectbox/DashboardSelectbox.vue b/cost-fe/src/components/pages/dashboard/dashboard-selectbox/DashboardSelectbox.vue index 5334c12..4ba1196 100644 --- a/cost-fe/src/components/pages/dashboard/dashboard-selectbox/DashboardSelectbox.vue +++ b/cost-fe/src/components/pages/dashboard/dashboard-selectbox/DashboardSelectbox.vue @@ -1,74 +1,68 @@ @@ -93,9 +87,12 @@ export default { workspace: { 'workspaceNM': 'WorkSpace' }, - project: [], // 여러 개의 프로젝트를 선택할 수 있도록 배열로 변경 - csp: [] // 여러 개의 CSP를 선택할 수 있도록 배열로 변경 - } + project: [], + csp: [] + }, + tumblebugWorkspaceid: null, + tumblebugProjectid: [], + tumblebugUsertoken: null }; }, setup() { @@ -134,32 +131,28 @@ export default { this.$emit('selectOptions') } if (type === 'project') { - // console.log('선택한 프로젝트 : ', this.selectedOptions); const index = this.selectedOptions.project.findIndex(proj => proj.projectCD === option.projectCD); if (index > -1) { - // 이미 선택된 경우 해제 this.selectedOptions.project.splice(index, 1); } else { - // 선택되지 않은 경우 추가 this.selectedOptions.project.push(option); } this.store.toggleProject(this.selectedOptions.project); - this.$emit('selectOptions') + this.$emit('selectOptions') } else if (type === 'csp') { const index = this.selectedOptions.csp.indexOf(option); + if (index > -1) { - // 이미 선택된 경우 해제 this.selectedOptions.csp.splice(index, 1); } else { - // 선택되지 않은 경우 추가 this.selectedOptions.csp.push(option); } + this.store.toggleCSP(this.selectedOptions.csp); this.$emit('selectOptions') } else { this.selectedOptions[type] = option; } - console.log('this.store: ', this.store.selectedOptions) this.resetOptions(); }, isSelected(type, option) { @@ -178,12 +171,9 @@ export default { .then(response => { if (response.data.status === 'OK') { this.workspaceOptions = response.data.Data; - // 초기 workspace 선택 if (this.workspaceOptions.length > 0) { this.selectOption('workspace', this.workspaceOptions[0]); } - } else { - console.error('api 호출 실패: ', response.data); } }) .catch(error => { @@ -199,6 +189,9 @@ export default { .then(response => { if (response.data.status === "OK") { this.projectOptions = response.data.Data; + if (this.projectOptions.length > 0) { + this.selectOption('project', this.projectOptions[0]); + } } else { console.error('api 호출 실패: ', response.data); } @@ -206,114 +199,136 @@ export default { .catch(error => { console.error(error); }); + }, + handleMessage(event){ + const projectCode = event.data && event.data.projectid !== undefined; + if (projectCode) { + this.tumblebugWorkspaceid = event.data.workspaceid; + this.store.setTumblebugWorkspace(this.tumblebugWorkspaceid); + this.tumblebugProjectid = event.data.projectid; + this.store.setTumblebugProject(this.tumblebugProjectid); + this.tumblebugUsertoken = event.data.usertoken; + this.store.setTumblebugUserToken(this.tumblebugUsertoken); + this.$emit('selectOptions'); + } else { + alert('프로젝트 코드를 전달받지 못했습니다.') + this.tumblebugWorkspaceid = 'testWs'; + this.store.setTumblebugWorkspace(this.tumblebugWorkspaceid); + this.tumblebugProjectid = ['testPrj']; + this.store.setTumblebugProject(this.tumblebugProjectid); + this.tumblebugUsertoken = 'Null'; + this.store.setTumblebugUserToken(this.tumblebugUsertoken); + this.$emit('selectOptions'); } + } }, mounted() { - this.fetchWorkspaces(); + window.addEventListener('message', this.handleMessage); + // this.fetchWorkspaces(); } } - + diff --git a/cost-fe/src/components/pages/dashboard/dashboard-top-resources/DashboardTop5.vue b/cost-fe/src/components/pages/dashboard/dashboard-top-resources/DashboardTop5.vue index 306f5c1..214b1df 100644 --- a/cost-fe/src/components/pages/dashboard/dashboard-top-resources/DashboardTop5.vue +++ b/cost-fe/src/components/pages/dashboard/dashboard-top-resources/DashboardTop5.vue @@ -1,27 +1,26 @@ - + diff --git a/cost-fe/src/components/pages/userGuide/Alarm.vue b/cost-fe/src/components/pages/userGuide/Alarm.vue index 056babc..0f75bd9 100644 --- a/cost-fe/src/components/pages/userGuide/Alarm.vue +++ b/cost-fe/src/components/pages/userGuide/Alarm.vue @@ -240,7 +240,7 @@ export default { getInitSlackInfo(){ axios.get(ENDPOINT.alaram + '/alert/getSlackIF', { params: { - userId: '2194155' + userId: 'test' }}) .then((res) => { if(res.data.status === 'OK'){ diff --git a/cost-fe/src/components/pages/userGuide/UserGuideLayout.vue b/cost-fe/src/components/pages/userGuide/UserGuideLayout.vue index 752d7a7..bdeba06 100644 --- a/cost-fe/src/components/pages/userGuide/UserGuideLayout.vue +++ b/cost-fe/src/components/pages/userGuide/UserGuideLayout.vue @@ -3,8 +3,11 @@
+ - +
@@ -14,18 +17,24 @@ import DashboardHeader from "@/components/pages/dashboard/dashboard-header/DashboardHeader.vue"; import Alarm from "@/components/pages/userGuide/Alarm.vue"; import AlarmGuideGrid from "@/components/pages/userGuide/guideGrid/AlarmGuideGrid.vue"; +import DashboardSelectbox from "@/components/pages/dashboard/dashboard-selectbox/DashboardSelectbox.vue"; export default { name: 'BillingInvoiceLayout', components: { DashboardHeader, Alarm, - AlarmGuideGrid + AlarmGuideGrid, + DashboardSelectbox }, data() { return {}; }, - methods: {} + methods: { + getWidgetData(){ + this.$refs.alarmGrid.fetchAlarmHistory(); + } + } } diff --git a/cost-fe/src/components/pages/userGuide/guideGrid/AlarmGuideGrid.vue b/cost-fe/src/components/pages/userGuide/guideGrid/AlarmGuideGrid.vue index 49d6bec..1491fed 100644 --- a/cost-fe/src/components/pages/userGuide/guideGrid/AlarmGuideGrid.vue +++ b/cost-fe/src/components/pages/userGuide/guideGrid/AlarmGuideGrid.vue @@ -2,342 +2,176 @@
-

Invoices

+

추천 이력

-
-
-
- Show -
- -
- entries -
-
- Search: -
- -
-
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
No. - - Invoice SubjectClientVAT No.CreatedStatusPrice
001401Design Works - - Carlson Limited - - 87956621 - - 15 Dec 2017 - - Paid - $887 - - -
- - Action - - - Another action - -
- -
001402UX Wireframes - - Adobe - - 87956421 - - 12 Apr 2017 - - Pending - $1200 - - -
- - Action - - - Another action - -
- -
001403New Dashboard - - Bluewolf - - 87952621 - - 23 Oct 2017 - - Pending - $534 - - -
- - Action - - - Another action - -
- -
001404Landing Page - - Salesforce - - 87953421 - - 2 Sep 2017 - - Due in 2 Weeks - $1500 - - -
- - Action - - - Another action - -
- -
001405Marketing Templates - - Printic - - 87956621 - - 29 Jan 2018 - - Paid Today - $648 - - -
- - Action - - - Another action - -
- -
001406Sales Presentation - - Tabdaq - - 87956621 - - 4 Feb 2018 - - Due in 3 Weeks - $300 - - -
- - Action - - - Another action - -
- -
001407Logo & Print - - Apple - - 87956621 - - 22 Mar 2018 - - Paid Today - $2500 - - -
- - Action - - - Another action - -
- -
001408Icons - - Tookapic - - 87956621 - - 13 May 2018 - - Paid Today - $940 - - -
- - Action - - - Another action - -
- -
-
-
-

Showing 1 to 8 of 16 entries

- +
+ + + + + + + + + + + + + + + + + + + + +
diff --git a/cost-fe/src/main.js b/cost-fe/src/main.js index b1c639f..2ccf87f 100644 --- a/cost-fe/src/main.js +++ b/cost-fe/src/main.js @@ -2,11 +2,19 @@ import { createApp } from 'vue' import App from './App.vue' import router from './router' import { createPinia } from 'pinia' +import PrimeVue from 'primevue/config' // Bootstrap CSS, JavaScript // import 'bootstrap/dist/css/bootstrap.min.css'; import 'bootstrap/dist/js/bootstrap.bundle.min.js'; +// PrimeVue styles +import 'primevue/resources/themes/saga-blue/theme.css' // theme +import 'primevue/resources/primevue.min.css' // core CSS +import 'primeicons/primeicons.css' // icons -createApp(App).use(router).use(createPinia()).mount('#app') +const app = createApp(App) +app.use(router).use(createPinia()).use(PrimeVue) + +app.mount('#app') \ No newline at end of file diff --git a/cost-fe/src/stores/selectedOptions.js b/cost-fe/src/stores/selectedOptions.js index 307961b..a927d71 100644 --- a/cost-fe/src/stores/selectedOptions.js +++ b/cost-fe/src/stores/selectedOptions.js @@ -7,6 +7,7 @@ export const useSelectedOptionsStore = defineStore('selectedOptions', { selectedWorkspace: "", selectedProjects: [], // 여러 개의 프로젝트를 선택할 수 있도록 배열로 변경 selectedCsps: [], // 여러 개의 CSP를 선택할 수 있도록 배열로 변경 + selectedUserToekn: "", today: ps.date.toFormatString('','yyyymmdd') } }), @@ -22,6 +23,15 @@ export const useSelectedOptionsStore = defineStore('selectedOptions', { }, toggleCSP(csp) { this.selectedOptions.selectedCsps = csp; + }, + setTumblebugWorkspace(workspace) { + this.selectedOptions.selectedWorkspace = workspace; + }, + setTumblebugProject(projects){ + this.selectedOptions.selectedProjects = projects; + }, + setTumblebugUserToken(userToken){ + this.selectedOptions.selectedUserToekn = userToken; } } }); diff --git a/costCollector/pom.xml b/costCollector/pom.xml index 222d038..8977dc5 100644 --- a/costCollector/pom.xml +++ b/costCollector/pom.xml @@ -95,6 +95,39 @@ log4jdbc-spring-boot-starter 2.0.0 + + junit + junit + test + + + + software.amazon.awssdk + bom + 2.26.30 + pom + import + + + software.amazon.awssdk + sts + 2.26.30 + + + software.amazon.awssdk + s3 + 2.26.30 + + + software.amazon.awssdk + costexplorer + 2.26.30 + + + org.apache.commons + commons-compress + 1.26.2 + diff --git a/costCollector/src/main/java/com/mcmp/collector/batch/CurCollect.java b/costCollector/src/main/java/com/mcmp/collector/batch/CurCollect.java new file mode 100644 index 0000000..484f6e7 --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/batch/CurCollect.java @@ -0,0 +1,88 @@ +package com.mcmp.collector.batch; + +import com.mcmp.collector.dao.AwsDao; +import com.mcmp.collector.model.cur.CurProcessModel; +import com.mcmp.collector.service.Cur; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobScope; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.List; + +@Slf4j +@Configuration +@RequiredArgsConstructor +public class CurCollect { + + @Autowired + private Cur cur; + + @Autowired + private AwsDao awsDao; + + @Bean + public Job curCollectJob(JobRepository jobRepository, Step curCollectStep) { + return new JobBuilder("curCollectJob", jobRepository) + .start(curCollectStep) + .build(); + } + + @Bean + @JobScope + public Step curCollectStep(JobRepository jobRepository, Tasklet curTasklet, PlatformTransactionManager platformTransactionManager){ + return new StepBuilder("curCollectStep", jobRepository) + .tasklet(curTasklet, platformTransactionManager).build(); + } + + @Bean + @StepScope + public Tasklet curTasklet(@Value("#{jobParameters['seq']}") String seq){ + return ((contribution, chunkContext) -> { + log.info(">>>START CUR BATCH JOB"); + + LocalDateTime collectDt = LocalDateTime.now(); + + List payers = awsDao.getPayerID(); + int run_idx = 0; + for(String payer : payers){ + + if(collectDt.getDayOfMonth() == 1 && seq.equals("1")){ + CurProcessModel initCURProcess = CurProcessModel.builder() + .csp("AWS") + .payer_account(payer) + .collect_date(String.format("%04d%02d", collectDt.getYear(), collectDt.getMonthValue())) + .certifed_fixed_yn("N") + .build(); + + awsDao.insertCURProcess(initCURProcess); + } + + List todos = awsDao.getTodoCURCollectMonth(payer); + + for(CurProcessModel todo : todos){ + cur.batchInsertCURData(payer, collectDt, seq, todo.getCollect_date(), todo.getObject_key(), run_idx); + run_idx += run_idx; + } + + } + + return RepeatStatus.FINISHED; + }); + } +} diff --git a/costCollector/src/main/java/com/mcmp/collector/batch/UnusedCollect.java b/costCollector/src/main/java/com/mcmp/collector/batch/UnusedCollect.java index 744b9d6..dae82b2 100644 --- a/costCollector/src/main/java/com/mcmp/collector/batch/UnusedCollect.java +++ b/costCollector/src/main/java/com/mcmp/collector/batch/UnusedCollect.java @@ -34,10 +34,12 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.BadSqlGrammarException; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.transaction.PlatformTransactionManager; import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.List; import java.util.concurrent.Future; import java.util.stream.Collectors; @@ -122,9 +124,18 @@ public ItemProcessor> processor(LocalDateTi param.setAccount(item.getAccount()); param.setStartDt(startDatetime); param.setEndDt(endDatetime); - - List data = unusedDao.getResourceSet(param); - + param.setYearMonth(startDatetime.format(DateTimeFormatter.ofPattern("yyyyMM"))); + + List data = null; + try{ + data = unusedDao.getResourceSet(param); + }catch (BadSqlGrammarException ex){ + if(isTableNotFound(ex)){ + log.warn("[Unused Log]NotFoundTable : {}", ex.getMessage()); + }else{ + log.error("[Unused Log] Occure Processor Error : {}", ex.getMessage()); + } + } return data; }; } @@ -161,7 +172,7 @@ public ItemWriter> writerTask2() { return itemLists -> { for (List itemList : itemLists) { for(ResourceSetModel item : itemList){ - System.out.println("Writer: 쓴다 " + item); +// System.out.println("Writer: 쓴다 " + item); } } }; @@ -188,4 +199,11 @@ public AsyncItemWriter> asyncItemWriter(){ return asyncItemWriter; } + + private boolean isTableNotFound(BadSqlGrammarException ex){ + String msg = ex.getCause().getMessage(); + + return msg != null && msg.contains("Table") && (msg.contains("doesn't exist") || msg.contains("not found") || + msg.contains("does not exist") || msg.contains("ORA-00942")); + } } diff --git a/costCollector/src/main/java/com/mcmp/collector/config/DatabaseConfig.java b/costCollector/src/main/java/com/mcmp/collector/config/DatabaseConfig.java index fe15bb3..8146bc0 100644 --- a/costCollector/src/main/java/com/mcmp/collector/config/DatabaseConfig.java +++ b/costCollector/src/main/java/com/mcmp/collector/config/DatabaseConfig.java @@ -46,7 +46,7 @@ public SqlSessionFactory sqlSessionBatch() throws Exception { public SqlSessionFactory sqlSessionSimple() throws Exception { SqlSessionFactoryBean sqlSession = new SqlSessionFactoryBean(); sqlSession.setDataSource(dataSourceBatch()); - sqlSession.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(ResourcePatternResolver.CLASSPATH_URL_PREFIX + "/mapper/unused/*_SQL.xml")); + sqlSession.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(ResourcePatternResolver.CLASSPATH_URL_PREFIX + "/mapper/**/*_SQL.xml")); return sqlSession.getObject(); } diff --git a/costCollector/src/main/java/com/mcmp/collector/dao/AwsDao.java b/costCollector/src/main/java/com/mcmp/collector/dao/AwsDao.java new file mode 100644 index 0000000..fd2a7cb --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/dao/AwsDao.java @@ -0,0 +1,71 @@ +package com.mcmp.collector.dao; + +import com.mcmp.collector.model.aws.DataExportBucketModel; +import com.mcmp.collector.model.aws.UserArnModel; +import com.mcmp.collector.model.cur.AwsCurDetailModel; +import com.mcmp.collector.model.cur.AwsCurModel; +import com.mcmp.collector.model.cur.CurProcessModel; +import com.mcmp.collector.model.cur.RscGrpMetaModel; +import org.mybatis.spring.SqlSessionTemplate; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +import java.util.List; + +@Repository +public class AwsDao { + + @Autowired + @Qualifier("sqlSessionTemplateSimple") + private SqlSessionTemplate sqlSessionTemplate; + + public void createTable(String suffix){ + sqlSessionTemplate.update("aws.createTable", suffix); + } + + public void dropTable(String suffix){ + sqlSessionTemplate.update("aws.dropTable", suffix); + } + + public void insertCURProcess(CurProcessModel model){ + sqlSessionTemplate.insert("aws.insertCURProcess", model); + } + + public List getTodoCURCollectMonth(String account){ + return sqlSessionTemplate.selectList("aws.getTodoCURCollectMonth", account); + } + + public List getPayerID(){ + return sqlSessionTemplate.selectList("aws.getPayerID"); + } + + public UserArnModel getUserArn(String cmpUserId){ + return sqlSessionTemplate.selectOne("aws.getUserArn", cmpUserId); + } + + public DataExportBucketModel getExportBucket(String cmpUserId){ + return sqlSessionTemplate.selectOne("aws.getDataExportBucket", cmpUserId); + } + + public void insertCurOriginBatch(List batchList){ + sqlSessionTemplate.insert("aws.insertCurOriginBatch", batchList); + } + + public void insertCurDetailBatch(AwsCurDetailModel model){ + sqlSessionTemplate.insert("aws.insertCurDetailBatch", model); + } + + public void insertRscGrpMeta(RscGrpMetaModel model){ + sqlSessionTemplate.insert("aws.insertRscGrpMeta", model); + } + + public void insertMonthlySum(String yearmonth){ + sqlSessionTemplate.insert("aws.insertMonthlySum", yearmonth); + } + + public void insertDailySumByProduct(String yearmonth){ + sqlSessionTemplate.insert("aws.insertDailySumByProduct", yearmonth); + } + +} diff --git a/costCollector/src/main/java/com/mcmp/collector/model/aws/DataExportBucketModel.java b/costCollector/src/main/java/com/mcmp/collector/model/aws/DataExportBucketModel.java new file mode 100644 index 0000000..30aa0c7 --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/model/aws/DataExportBucketModel.java @@ -0,0 +1,8 @@ +package com.mcmp.collector.model.aws; + +import lombok.Data; + +@Data +public class DataExportBucketModel extends UserArnModel{ + private String butcket_name; +} diff --git a/costCollector/src/main/java/com/mcmp/collector/model/aws/UserArnModel.java b/costCollector/src/main/java/com/mcmp/collector/model/aws/UserArnModel.java new file mode 100644 index 0000000..f24c247 --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/model/aws/UserArnModel.java @@ -0,0 +1,11 @@ +package com.mcmp.collector.model.aws; + +import lombok.Data; + +@Data +public class UserArnModel { + + private String mcmp_id; + private String csp; + private String role_arn; +} diff --git a/costCollector/src/main/java/com/mcmp/collector/model/cur/AwsCurDetailModel.java b/costCollector/src/main/java/com/mcmp/collector/model/cur/AwsCurDetailModel.java new file mode 100644 index 0000000..0bd3684 --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/model/cur/AwsCurDetailModel.java @@ -0,0 +1,11 @@ +package com.mcmp.collector.model.cur; + +import lombok.Data; + +import java.util.List; + +@Data +public class AwsCurDetailModel { + String suffix; + List batchList; +} diff --git a/costCollector/src/main/java/com/mcmp/collector/model/cur/AwsCurModel.java b/costCollector/src/main/java/com/mcmp/collector/model/cur/AwsCurModel.java new file mode 100644 index 0000000..c028d04 --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/model/cur/AwsCurModel.java @@ -0,0 +1,89 @@ +package com.mcmp.collector.model.cur; + +import lombok.Data; + +import java.sql.Timestamp; +import java.time.LocalDateTime; + +@Data +public class AwsCurModel { + + private String lineitem_usageaccountid; + private String lineitem_productcode; + private String lineitem_resourceid; + private String lineitem_lineitemtype; + private String product_instancetype; + private String pricing_unit; + private LocalDateTime lineitem_usagestartdate; + private LocalDateTime lineitem_usageenddate; + private String lineitem_usageamount; + private String lineitem_unblendedcost; + private String lineitem_blendedcost; + private String pricing_publicondemandcost; + private String pricing_publicondemandrate; + private String lineitem_currencycode; + private LocalDateTime collect_date; + private String seq; + private String product_sku; + private String product_region; + private String product_instanceFamily; + private String product_location; + private String lineitem_operation; + private String product_instancetypefamily; + private String lineitem_usagetype; + private String product_vcpu; + private String product_memory; + + public AwsCurModel(String lineitem_usageaccountid, + String lineitem_productcode, + String lineitem_resourceid, + String lineitem_lineitemtype, + String product_instancetype, + String pricing_unit, + String lineitem_usageamount, + String lineitem_unblendedcost, + String lineitem_blendedcost, + LocalDateTime lineitem_usagestartdate, + LocalDateTime lineitem_usageenddate, + String pricing_publicondemandcost, + String pricing_publicondemandrate, + String lineitem_currencycode, + LocalDateTime collect_date, + String seq, + String product_sku, + String product_region, + String product_instanceFamily, + String product_location, + String lineitem_operation, + String product_instancetypefamily, + String lineitem_usagetype, + String product_vcpu, + String product_memory){ + this.lineitem_usageaccountid = lineitem_usageaccountid; + this.lineitem_productcode = lineitem_productcode; + this.lineitem_resourceid = lineitem_resourceid; + this.lineitem_lineitemtype = lineitem_lineitemtype; + this.product_instancetype = product_instancetype; + this.pricing_unit = pricing_unit; + this.lineitem_usageamount = lineitem_usageamount; + this.lineitem_unblendedcost = lineitem_unblendedcost; + this.lineitem_blendedcost = lineitem_blendedcost; + this.lineitem_usagestartdate = lineitem_usagestartdate; + this.lineitem_usageenddate = lineitem_usageenddate; + this.pricing_publicondemandcost = pricing_publicondemandcost; + this.pricing_publicondemandrate = pricing_publicondemandrate; + this.lineitem_currencycode = lineitem_currencycode; + this.collect_date = collect_date; + this.seq = seq; + this.product_sku = product_sku; + this.product_region = product_region; + this.product_instanceFamily = product_instanceFamily; + this.product_location = product_location; + this.lineitem_operation = lineitem_operation; + this.product_instancetypefamily = product_instancetypefamily; + this.lineitem_usagetype = lineitem_usagetype; + this.product_vcpu = product_vcpu; + this.product_memory = product_memory; + } + +} diff --git a/costCollector/src/main/java/com/mcmp/collector/model/cur/CurProcessModel.java b/costCollector/src/main/java/com/mcmp/collector/model/cur/CurProcessModel.java new file mode 100644 index 0000000..328ba5f --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/model/cur/CurProcessModel.java @@ -0,0 +1,21 @@ +package com.mcmp.collector.model.cur; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class CurProcessModel { + private String csp; + private String payer_account; + private String collect_date; + private String certifed_fixed_yn; + private LocalDateTime certifed_fixed_date; + private String object_key; +} diff --git a/costCollector/src/main/java/com/mcmp/collector/model/cur/RscGrpMetaModel.java b/costCollector/src/main/java/com/mcmp/collector/model/cur/RscGrpMetaModel.java new file mode 100644 index 0000000..fe25dff --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/model/cur/RscGrpMetaModel.java @@ -0,0 +1,13 @@ +package com.mcmp.collector.model.cur; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class RscGrpMetaModel { + private String csp; + private String account; + private String prj_cd; + private String year_month; +} diff --git a/costCollector/src/main/java/com/mcmp/collector/model/unused/ResourceSetModel.java b/costCollector/src/main/java/com/mcmp/collector/model/unused/ResourceSetModel.java index c3f4dbd..1cf7fed 100644 --- a/costCollector/src/main/java/com/mcmp/collector/model/unused/ResourceSetModel.java +++ b/costCollector/src/main/java/com/mcmp/collector/model/unused/ResourceSetModel.java @@ -8,4 +8,12 @@ public class ResourceSetModel { private String instanceid; private Timestamp createDt; + private String resourceType; + private String instanceType; + private String regionId; + private String regionNM; + private String operation; + private String instanceFamily; + private String instanceFamilyType; + private String productSKU; } diff --git a/costCollector/src/main/java/com/mcmp/collector/model/unused/ResourceSetParamModel.java b/costCollector/src/main/java/com/mcmp/collector/model/unused/ResourceSetParamModel.java index ffef7b5..5956cfd 100644 --- a/costCollector/src/main/java/com/mcmp/collector/model/unused/ResourceSetParamModel.java +++ b/costCollector/src/main/java/com/mcmp/collector/model/unused/ResourceSetParamModel.java @@ -10,4 +10,5 @@ public class ResourceSetParamModel { private LocalDateTime startDt; private LocalDateTime endDt; private String account; + private String yearMonth; } diff --git a/costCollector/src/main/java/com/mcmp/collector/schedule/CurScheduleJob.java b/costCollector/src/main/java/com/mcmp/collector/schedule/CurScheduleJob.java new file mode 100644 index 0000000..15b2736 --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/schedule/CurScheduleJob.java @@ -0,0 +1,61 @@ +package com.mcmp.collector.schedule; + +import lombok.extern.slf4j.Slf4j; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.scheduling.quartz.QuartzJobBean; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; + +@Component +@Slf4j +public class CurScheduleJob extends QuartzJobBean { + + @Autowired + private JobLauncher jobLauncher; + + @Qualifier("curCollectJob") + @Autowired + private Job curCollectJob; + + @Override + protected void executeInternal(JobExecutionContext context) throws JobExecutionException { + log.info("Starting CUR Quartz Job"); + + try { + LocalDateTime now = LocalDateTime.now(); + + String seq; + if(now.getHour() == 0){ + seq = "1"; + } else if (now.getHour() == 6){ + seq = "2"; + } else { + seq = "0"; + } + + JobParameters jobParameter = new JobParametersBuilder() + .addString("batchType", "curCollect") + .addString("seq", seq) + .addLong("createTime", System.currentTimeMillis()) + .toJobParameters(); + + jobLauncher.run(curCollectJob, jobParameter); + } catch (Exception e) { + log.error("Error executing Cur Batch Job", e); + throw new JobExecutionException("Failed to execute Cur Batch Job", e); + } + + log.info("Quartz Job (Cur Batch) completed"); + } +} diff --git a/costCollector/src/main/java/com/mcmp/collector/schedule/ScheduleConfig.java b/costCollector/src/main/java/com/mcmp/collector/schedule/ScheduleConfig.java index 3ba47aa..68a1db6 100644 --- a/costCollector/src/main/java/com/mcmp/collector/schedule/ScheduleConfig.java +++ b/costCollector/src/main/java/com/mcmp/collector/schedule/ScheduleConfig.java @@ -12,6 +12,9 @@ public class ScheduleConfig { @Value("${unusedBatchCronSchedule}") public String batchCron ; + @Value("${curBatchCronSchedule}") + public String curBatchCron; + @Bean public JobDetail UnusedJobDetail() { return JobBuilder.newJob().ofType(ScheduleJob.class) @@ -31,4 +34,24 @@ public Trigger sampleJobTrigger(JobDetail UnusedJobDetail) { .withSchedule(cronSchedule) .build(); } + + @Bean + public JobDetail CurJobDetail() { + return JobBuilder.newJob().ofType(CurScheduleJob.class) + .storeDurably() + .withIdentity("CurJob", "Cur") + .withDescription("Execute Spring Batch Cur Batch Job with Quartz") + .build(); + } + + @Bean + public Trigger CurJobTrigger(JobDetail CurJobDetail) { + CronScheduleBuilder cronSchedule = CronScheduleBuilder.cronSchedule(curBatchCron); + + return TriggerBuilder.newTrigger().forJob(CurJobDetail) + .withIdentity("CurJobTrigger1", "Cur") + .withDescription("Unused Job Trigger") + .withSchedule(cronSchedule) + .build(); + } } diff --git a/costCollector/src/main/java/com/mcmp/collector/schedule/ScheduleJob.java b/costCollector/src/main/java/com/mcmp/collector/schedule/ScheduleJob.java index 00eefcb..86187f3 100644 --- a/costCollector/src/main/java/com/mcmp/collector/schedule/ScheduleJob.java +++ b/costCollector/src/main/java/com/mcmp/collector/schedule/ScheduleJob.java @@ -32,9 +32,9 @@ protected void executeInternal(JobExecutionContext context) throws JobExecutionE log.info("Starting Quartz Job"); try { - ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC")); - String startOfToday = String.valueOf(now.toLocalDate().atStartOfDay()); - String endOfToday = String.valueOf(now.toLocalDate().plusDays(1).atStartOfDay()); + ZonedDateTime now = ZonedDateTime.now(); + String endOfToday = String.valueOf(now.toLocalDate().atStartOfDay()); + String startOfToday = String.valueOf(now.toLocalDate().minusDays(1).atStartOfDay()); JobParameters jobParameter = new JobParametersBuilder() .addString("startDt", startOfToday) diff --git a/costCollector/src/main/java/com/mcmp/collector/service/Abnormal.java b/costCollector/src/main/java/com/mcmp/collector/service/Abnormal.java new file mode 100644 index 0000000..dc9dbd2 --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/service/Abnormal.java @@ -0,0 +1,48 @@ +package com.mcmp.collector.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.costexplorer.CostExplorerClient; +import software.amazon.awssdk.services.costexplorer.model.AnomalyDateInterval; +import software.amazon.awssdk.services.costexplorer.model.GetAnomaliesRequest; +import software.amazon.awssdk.services.costexplorer.model.GetAnomaliesResponse; + +@Slf4j +@Service +public class Abnormal { + + @Autowired + private AssumeRole assumeRole; + + public void getCostAnomaly(String payerId){ + StaticCredentialsProvider staticCredentialsProvider = assumeRole.assumeRole(payerId); + + try(CostExplorerClient costExplorerClient = CostExplorerClient.builder() + .credentialsProvider(staticCredentialsProvider) + .region(Region.US_EAST_1) // 현재 여기에서만 cost and billing 확인 가능 + .build()){ + GetAnomaliesRequest anomaliesRequest = GetAnomaliesRequest.builder() + .dateInterval(AnomalyDateInterval.builder() + .startDate("2024-08-30") + .endDate("2024-09-20") + .build()) + .build(); + + GetAnomaliesResponse anomaliesResponse = costExplorerClient.getAnomalies(anomaliesRequest); + + System.out.println("이상비용탐지 api 결과 : " + anomaliesResponse); + + anomaliesResponse.anomalies().forEach(anomaly -> { + System.out.println("Anomaly ID: " + anomaly.anomalyId()); + System.out.println("Anomaly Score: " + anomaly.anomalyScore()); + System.out.println("Impact: " + anomaly.impact()); + }); + + } catch (Exception e){ + e.printStackTrace(); + } + } +} diff --git a/costCollector/src/main/java/com/mcmp/collector/service/AssumeRole.java b/costCollector/src/main/java/com/mcmp/collector/service/AssumeRole.java new file mode 100644 index 0000000..86c3e47 --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/service/AssumeRole.java @@ -0,0 +1,81 @@ +package com.mcmp.collector.service; + +import com.mcmp.collector.dao.AwsDao; +import com.mcmp.collector.model.aws.DataExportBucketModel; +import com.mcmp.collector.model.aws.UserArnModel; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; +import software.amazon.awssdk.services.sts.model.Credentials; +import software.amazon.awssdk.services.sts.model.StsException; + +import java.time.Instant; + +@Service +@Slf4j +public class AssumeRole { + + @Autowired + private AwsDao awsDao; + + + private final String roleSessionName = "s3-assume-role-" + Instant.now().toEpochMilli(); + private final Region region = Region.AP_NORTHEAST_2; + + public StaticCredentialsProvider assumeRole(String cmpUserId){ + try{ + UserArnModel userModel = awsDao.getUserArn(cmpUserId); + if(userModel == null || userModel.getRole_arn() == null){ + log.error("No user found with cmpUserId: {}", cmpUserId); + return null; + } + + Credentials tempRoleCredentials; + try( StsClient stsClient = StsClient.builder() + .region(region).build()){ + AssumeRoleRequest roleRequest = AssumeRoleRequest.builder() + .roleArn(userModel.getRole_arn()) + .roleSessionName(roleSessionName) + .build(); + AssumeRoleResponse roleResponse = stsClient.assumeRole(roleRequest); + tempRoleCredentials = roleResponse.credentials(); + } + + String key = tempRoleCredentials.accessKeyId(); + String secKey = tempRoleCredentials.secretAccessKey(); + String secToken = tempRoleCredentials.sessionToken(); + + StaticCredentialsProvider staticCredentialsProvider = StaticCredentialsProvider.create( + AwsSessionCredentials.create(key, secKey, secToken) + ); + + return staticCredentialsProvider; + + } catch (StsException | S3Exception e){ + log.error(e.getMessage()); + throw new RuntimeException(); + } catch (Exception e){ + e.printStackTrace(); + throw new RuntimeException(); + } + } + + public String getDataExportBucketNM(String cmpUserID){ + DataExportBucketModel bucketModel = awsDao.getExportBucket(cmpUserID); + + if(bucketModel != null){ + return bucketModel.getButcket_name(); + }else { + log.error("No Bucket Name found - User : " + cmpUserID); + throw new RuntimeException(); + } + } + +} diff --git a/costCollector/src/main/java/com/mcmp/collector/service/Cur.java b/costCollector/src/main/java/com/mcmp/collector/service/Cur.java new file mode 100644 index 0000000..1921e13 --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/service/Cur.java @@ -0,0 +1,330 @@ +package com.mcmp.collector.service; + +import com.mcmp.collector.dao.AwsDao; +import com.mcmp.collector.model.cur.AwsCurDetailModel; +import com.mcmp.collector.model.cur.AwsCurModel; +import com.mcmp.collector.model.cur.CurProcessModel; +import com.mcmp.collector.model.cur.RscGrpMetaModel; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.services.sts.model.StsException; + +import java.io.*; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; + +@Service +@Slf4j +public class Cur { + @Autowired + private AssumeRole assumeRole; + + @Autowired + private DateCalculator dateCalculator; + + @Autowired + private AwsDao awsDao; + + @Value("${aws.data.export.name}") + private String dataExportName; + + @Value("${aws.data.export.path.prefix}") + private String dataExportPathPrefix; + + private static final Region region = Region.AP_NORTHEAST_2; + private static final int BATCH_SIZE = 50; + private static final DateTimeFormatter dtFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssX"); + + public void batchInsertCURData(String payerId, LocalDateTime collectDt, String seq, String todoCollectMonth, String preObjectKey, int run_idx){ + try{ + String bucketNM = assumeRole.getDataExportBucketNM(payerId); + + S3Object s3Object = null; + try { + s3Object = getFileKey(payerId, bucketNM, todoCollectMonth); + + if(Objects.equals(preObjectKey, s3Object.key()) || s3Object == null){ + log.warn("No new cur data is created - Payer : " + payerId + ", Month : " + todoCollectMonth); + return; + } + } catch (Exception e){ + e.printStackTrace(); + log.warn("No data for that month - Payer : " + payerId + ", Month : " + todoCollectMonth); + return; + } + + ResponseInputStream gzObject = getCsvGzObject(payerId, bucketNM, s3Object.key()); + + String line; + List batchList = new ArrayList<>(); + Map headerMap = new HashMap<>(); + + try(InputStream gzipInputStream = new GZIPInputStream(gzObject); + BufferedReader reader = new BufferedReader(new InputStreamReader(gzipInputStream))){ + + String headerLine = reader.readLine(); + if (headerLine != null) { + String[] headers = headerLine.split(","); + for (int i = 0; i < headers.length; i++) { + headerMap.put(headers[i].trim(), i); + } + } + + String[] keys = { + "lineItem/UsageAccountId", "lineItem/ProductCode", "lineItem/ResourceId", + "lineItem/LineItemType", "product/instanceType", "pricing/unit", + "lineItem/UsageAmount", "lineItem/UnblendedCost", "lineItem/BlendedCost", + "lineItem/UsageStartDate", "lineItem/UsageEndDate", "pricing/publicOnDemandCost", + "pricing/publicOnDemandRate", "lineItem/CurrencyCode", "bill/InvoiceId", "product/sku", "product/region", + "product/instanceFamily", "product/location", "lineItem/Operation", + "product/instanceTypeFamily", "lineItem/UsageType", "product/vcpu", "product/memory" + }; + Map indexMap = Arrays.stream(keys) + .collect(Collectors.toMap(key -> key, key -> headerMap.getOrDefault(key, -1))); + boolean allColumnsPresent = indexMap.values().stream().noneMatch(index -> index == -1); + if (!allColumnsPresent) { + log.error("Cur Batch Insert - One or more required headers are missing. Stop processing your data."); + return; + } + + if(run_idx == 0){ + awsDao.dropTable(todoCollectMonth); + awsDao.createTable(todoCollectMonth); + } + + String certifed_fixed_yn = "Y"; + + while ((line = reader.readLine()) != null) { + String[] fields = line.split(",", -1); + OffsetDateTime usageStartDt = OffsetDateTime.parse(fields[indexMap.get("lineItem/UsageStartDate")], dtFormatter); + OffsetDateTime usageEndDt = OffsetDateTime.parse(fields[indexMap.get("lineItem/UsageEndDate")], dtFormatter); + + if(fields[indexMap.get("bill/InvoiceId")].isEmpty()){ + certifed_fixed_yn = "N"; + } + + AwsCurModel usageData = new AwsCurModel( + fields[indexMap.get("lineItem/UsageAccountId")], + fields[indexMap.get("lineItem/ProductCode")], + fields[indexMap.get("lineItem/ResourceId")], + fields[indexMap.get("lineItem/LineItemType")], + fields[indexMap.get("product/instanceType")], + fields[indexMap.get("pricing/unit")], + fields[indexMap.get("lineItem/UsageAmount")], + fields[indexMap.get("lineItem/UnblendedCost")], + fields[indexMap.get("lineItem/BlendedCost")], + usageStartDt.toLocalDateTime(), + usageEndDt.toLocalDateTime(), + fields[indexMap.get("pricing/publicOnDemandCost")], + fields[indexMap.get("pricing/publicOnDemandRate")], + fields[indexMap.get("lineItem/CurrencyCode")], + collectDt, + seq, + fields[indexMap.get("product/sku")], + fields[indexMap.get("product/region")], + fields[indexMap.get("product/instanceFamily")], + fields[indexMap.get("product/location")], + fields[indexMap.get("lineItem/Operation")], + fields[indexMap.get("product/instanceTypeFamily")], + fields[indexMap.get("lineItem/UsageType")], + fields[indexMap.get("product/vcpu")], + fields[indexMap.get("product/memory")] + ); + + batchList.add(usageData); + + if (batchList.size() == BATCH_SIZE) { + try{ + insertOriginBatch(batchList); + + AwsCurDetailModel detailModel = new AwsCurDetailModel(); + detailModel.setSuffix(todoCollectMonth); + detailModel.setBatchList(batchList); + insertDetailBatch(detailModel); + } catch (Exception e){ + log.error("Error - Insert CUR Batch Data - Account : " + payerId); + e.printStackTrace(); + } + batchList.clear(); + } + } + + // Insert remaining records + if (!batchList.isEmpty()) { + insertOriginBatch(batchList); + AwsCurDetailModel detailModel = new AwsCurDetailModel(); + detailModel.setSuffix(todoCollectMonth); + detailModel.setBatchList(batchList); + insertDetailBatch(detailModel); + } + + CurProcessModel updateProcess = CurProcessModel.builder() + .csp("AWS") + .payer_account(payerId) + .collect_date(todoCollectMonth) + .certifed_fixed_yn(certifed_fixed_yn) + .object_key(s3Object.key()) + .build(); + if(certifed_fixed_yn.equals("Y")){ + updateProcess.setCertifed_fixed_date(collectDt); + } + + RscGrpMetaModel rscGrpMetaModel = RscGrpMetaModel.builder() + .csp("AWS") + .account(payerId) + .prj_cd("testPrj") + .year_month(todoCollectMonth) + .build(); + + awsDao.insertCURProcess(updateProcess); + awsDao.insertRscGrpMeta(rscGrpMetaModel); + + calMonthlySum(todoCollectMonth); + calDailySumByProduct(todoCollectMonth); + + } + + } catch (RuntimeException re) { + re.printStackTrace(); + } catch (Exception e){ + e.printStackTrace(); + } + } + + private void insertOriginBatch(List batchList) { + awsDao.insertCurOriginBatch(batchList); + } + + private void insertDetailBatch(AwsCurDetailModel model){ + awsDao.insertCurDetailBatch(model); + } + + public void getS3BucketNames(String payerId){ + StaticCredentialsProvider staticCredentialsProvider = assumeRole.assumeRole(payerId); + + try(S3Client s3 = S3Client.builder() + .credentialsProvider(staticCredentialsProvider) + .region(region) + .build()){ + List buckets = s3.listBuckets().buckets(); + for(Bucket bucket : buckets){ +// System.out.println("bucket name: " + bucket.name()); + } + } + } + + public S3Object getFileKey(String payerId, String bucketNM, String todoCollectMonth){ + StaticCredentialsProvider staticCredentialsProvider = assumeRole.assumeRole(payerId); + + Optional latestObject = Optional.empty(); + + try(S3Client s3Client = S3Client.builder() + .credentialsProvider(staticCredentialsProvider) + .region(region) + .build()){ + + ListObjectsV2Request listObjectsRequest = ListObjectsV2Request.builder() + .bucket(bucketNM) + .prefix(dataExportPathPrefix + "/" + dataExportName + "/" + dateCalculator.getMonthRange(todoCollectMonth, 1) + "/") + .build(); + + ListObjectsV2Response listObjectsResponse; + do { + listObjectsResponse = s3Client.listObjectsV2(listObjectsRequest); + + for (S3Object s3Object : listObjectsResponse.contents()) { + if ((!latestObject.isPresent() || s3Object.lastModified().isAfter(latestObject.get().lastModified())) && s3Object.key().contains(".csv")) { + latestObject = Optional.of(s3Object); + } + } + + listObjectsRequest = listObjectsRequest.toBuilder() + .continuationToken(listObjectsResponse.nextContinuationToken()) + .build(); + + } while (listObjectsResponse.isTruncated()); + + } catch (Exception e) { + e.printStackTrace(); + } + + return latestObject.orElseThrow(() -> new RuntimeException("No CUR Objects found")); + } + + public ResponseInputStream getCsvGzObject(String payerId, String bucketNM, String key){ + StaticCredentialsProvider staticCredentialsProvider = assumeRole.assumeRole(payerId); + + try{ + S3Client s3Client = S3Client.builder() + .credentialsProvider(staticCredentialsProvider) + .region(region) + .build(); + + GetObjectRequest objectRequest = GetObjectRequest.builder() + .bucket(bucketNM) + .key(key) + .build(); + + ResponseInputStream s3Object = s3Client.getObject(objectRequest); + + return s3Object; + + } catch (StsException e){ + e.printStackTrace(); + } + + throw new RuntimeException(); + } + + // Simple Duplicate s3 object to Local + public void getObjectBytes(String payerId){ + StaticCredentialsProvider staticCredentialsProvider = assumeRole.assumeRole(payerId); + + try{ + S3Client s3Client = S3Client.builder() + .credentialsProvider(staticCredentialsProvider) + .region(region) + .build(); + + GetObjectRequest objectRequest = GetObjectRequest.builder() + .bucket("mcmp-costopti-curbucket") + .key("s3 테스트 파일.txt") + .build(); + + ResponseBytes objectBytes = s3Client.getObjectAsBytes(objectRequest); + byte[] data = objectBytes.asByteArray(); + + File file = new File("C:\\txt"); + OutputStream os = new FileOutputStream(file); + os.write(data); + System.out.println("Successfully"); + os.close(); + + } catch (StsException e){ + e.printStackTrace(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public void calMonthlySum(String yearmonth){ + awsDao.insertMonthlySum(yearmonth); + } + + public void calDailySumByProduct(String yearmonth){ + awsDao.insertDailySumByProduct(yearmonth); + } + +} diff --git a/costCollector/src/main/java/com/mcmp/collector/service/DateCalculator.java b/costCollector/src/main/java/com/mcmp/collector/service/DateCalculator.java new file mode 100644 index 0000000..8593986 --- /dev/null +++ b/costCollector/src/main/java/com/mcmp/collector/service/DateCalculator.java @@ -0,0 +1,35 @@ +package com.mcmp.collector.service; + +import org.springframework.stereotype.Service; + +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; + +@Service +public class DateCalculator { + + private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd"); + + public String getCurMonthRange(){ + LocalDate now = LocalDate.now(ZoneOffset.UTC); + LocalDate firstDayofMonth = now.withDayOfMonth(1); + LocalDate firstDayofNextMonth = firstDayofMonth.plusMonths(1); + + String start = firstDayofMonth.format(FORMATTER); + String end = firstDayofNextMonth.format(FORMATTER); + + return start + "-" + end; + } + + public String getMonthRange(String month, int term){ + LocalDate date = LocalDate.parse(month+"01", DateTimeFormatter.ofPattern("yyyyMMdd")); + LocalDate firstDayofMonth = date.withDayOfMonth(1); + LocalDate firstDayofNextMonth = firstDayofMonth.plusMonths(term); + + String start = firstDayofMonth.format(FORMATTER); + String end = firstDayofNextMonth.format(FORMATTER); + + return start + "-" + end; + } +} diff --git a/costCollector/src/main/resources/application.properties b/costCollector/src/main/resources/application.properties index 44f5634..cac7c1d 100644 --- a/costCollector/src/main/resources/application.properties +++ b/costCollector/src/main/resources/application.properties @@ -4,7 +4,6 @@ server.port=8081 # Spring Batch Job enable spring.batch.job.enabled=false spring.batch.jdbc.initialize-schema=always -spring.batch.job.name=unusedCollectJob spring.datasource.hikari.batch.driver-class-name=org.mariadb.jdbc.Driver spring.datasource.hikari.batch.jdbc-url=jdbc:mariadb://localhost:3306/cost?autoReconnect=true&allowMultiQueries=true&useSSL=false&rewriteBatchedStatements=true @@ -18,6 +17,10 @@ spring.datasource.hikari.batch.remove-abandoned-on-borrow=true spring.datasource.hikari.batch.remove-abandoned-timeout=30 unusedBatchCronSchedule=0 30 0 * * ? +curBatchCronSchedule=0 0 0,6 * * ? logging.level.org.springframework.batch=DEBUG logging.level.org.mybatis=DEBUG + +aws.data.export.name=MCMP-CostOpti +aws.data.export.path.prefix=mcmp-costopti diff --git a/costCollector/src/main/resources/mapper/aws/AWS_SQL.xml b/costCollector/src/main/resources/mapper/aws/AWS_SQL.xml new file mode 100644 index 0000000..e6d9698 --- /dev/null +++ b/costCollector/src/main/resources/mapper/aws/AWS_SQL.xml @@ -0,0 +1,248 @@ + + + + + + DROP TABLE IF EXISTS tbl_table_billing_detail_${value} + + + + + + CREATE TABLE tbl_table_billing_detail_${value} ( + `lineitem_usageaccountid` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_productcode` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_resourceid` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_lineitemtype` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `product_instancetype` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `pricing_unit` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_usageamount` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_unblendedcost` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_blendedcost` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_usagestartdate` timestamp NULL DEFAULT NULL, + `lineitem_usageenddate` timestamp NULL DEFAULT NULL, + `pricing_publicondemandcost` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `pricing_publicondemandrate` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_currencycode` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `product_sku` varchar(100) DEFAULT NULL, + `product_region` varchar(50) DEFAULT NULL, + `product_instanceFamily` varchar(100) DEFAULT NULL, + `product_location` varchar(100) DEFAULT NULL, + `lineitem_operation` varchar(100) DEFAULT NULL, + `product_instancetypefamily` varchar(20) DEFAULT NULL, + `lineitem_usagetype` varchar(100) DEFAULT NULL, + `product_vcpu` varchar(30) DEFAULT NULL, + `product_memory` varchar(30) DEFAULT NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci + + + + + INSERT INTO cur_process_info(csp, payer_account, collect_date, certifed_fixed_yn, certifed_fixed_date, object_key) + VALUES (#{csp}, + #{payer_account}, + #{collect_date}, + #{certifed_fixed_yn}, + #{certifed_fixed_date}, + #{object_key}) + ON DUPLICATE KEY UPDATE certifed_fixed_yn=#{certifed_fixed_yn}, + certifed_fixed_date=#{certifed_fixed_date}, + object_key=#{object_key} + + + + + + + + + + + + INSERT INTO cur_origin(lineitem_usageaccountid, + lineitem_productcode, + lineitem_resourceid, + lineitem_lineitemtype, + product_instancetype, + pricing_unit, + lineitem_usageamount, + lineitem_unblendedcost, + lineitem_blendedcost, + lineitem_usagestartdate, + lineitem_usageenddate, + pricing_publicondemandcost, + pricing_publicondemandrate, + lineitem_currencycode, + data_collect_date, + data_collect_seq, + product_sku, + product_region, + product_instanceFamily, + product_location, + lineitem_operation, + product_instancetypefamily, + lineitem_usagetype, + product_vcpu, + product_memory) + VALUES + + (#{item.lineitem_usageaccountid}, + #{item.lineitem_productcode}, + #{item.lineitem_resourceid}, + #{item.lineitem_lineitemtype}, + #{item.product_instancetype}, + #{item.pricing_unit}, + #{item.lineitem_usageamount}, + #{item.lineitem_unblendedcost}, + #{item.lineitem_blendedcost}, + #{item.lineitem_usagestartdate}, + #{item.lineitem_usageenddate}, + #{item.pricing_publicondemandcost}, + #{item.pricing_publicondemandrate}, + #{item.lineitem_currencycode}, + #{item.collect_date}, + #{item.seq}, + #{item.product_sku}, + #{item.product_region}, + #{item.product_instanceFamily}, + #{item.product_location}, + #{item.lineitem_operation}, + #{item.product_instancetypefamily}, + #{item.lineitem_usagetype}, + #{item.product_vcpu}, + #{item.product_memory}) + + + + + INSERT INTO tbl_table_billing_detail_${suffix}(lineitem_usageaccountid, + lineitem_productcode, + lineitem_resourceid, + lineitem_lineitemtype, + product_instancetype, + pricing_unit, + lineitem_usageamount, + lineitem_unblendedcost, + lineitem_blendedcost, + lineitem_usagestartdate, + lineitem_usageenddate, + pricing_publicondemandcost, + pricing_publicondemandrate, + lineitem_currencycode, + product_sku, + product_region, + product_instanceFamily, + product_location, + lineitem_operation, + product_instancetypefamily, + lineitem_usagetype, + product_vcpu, + product_memory) + VALUES + + (#{item.lineitem_usageaccountid}, + #{item.lineitem_productcode}, + #{item.lineitem_resourceid}, + #{item.lineitem_lineitemtype}, + #{item.product_instancetype}, + #{item.pricing_unit}, + #{item.lineitem_usageamount}, + #{item.lineitem_unblendedcost}, + #{item.lineitem_blendedcost}, + #{item.lineitem_usagestartdate}, + #{item.lineitem_usageenddate}, + #{item.pricing_publicondemandcost}, + #{item.pricing_publicondemandrate}, + #{item.lineitem_currencycode}, + #{item.product_sku}, + #{item.product_region}, + #{item.product_instanceFamily}, + #{item.product_location}, + #{item.lineitem_operation}, + #{item.product_instancetypefamily}, + #{item.lineitem_usagetype}, + #{item.product_vcpu}, + #{item.product_memory}) + + + + + INSERT INTO servicegroup_meta(csp_type, csp_account, csp_instanceid, service_cd) + SELECT #{csp}, #{account}, lineitem_resourceid, #{prj_cd} + FROM tbl_table_billing_detail_${year_month} + GROUP BY lineitem_resourceid + ON DUPLICATE KEY UPDATE + csp_instanceid = values(csp_instanceid) + + + + INSERT INTO monthly_summation(`year_month` , project_cd, csp, total_cost) + SELECT ${value} + , sm.service_cd + , sm.csp_type + , COALESCE(SUM(ttbd.lineitem_unblendedcost), 0) + FROM tbl_table_billing_detail_${value} ttbd + LEFT JOIN servicegroup_meta sm + ON sm.csp_instanceid = ttbd.lineitem_resourceid + GROUP BY sm.csp_type + , sm.service_cd + ON DUPLICATE KEY update total_cost = values(total_cost) + + + + INSERT INTO daily_summation_by_product (`date`, product, total_cost, project_cd, csp_type) + SELECT DATE(ttbd.lineitem_usagestartdate) AS `date` + , ttbd.lineitem_productcode AS product + , COALESCE(SUM(ttbd.lineitem_unblendedcost), 0) AS total_cost + , sm.service_cd AS project_cd + , 'AWS' as csp_type + FROM tbl_table_billing_detail_${value} ttbd + LEFT JOIN servicegroup_meta sm + ON sm.csp_instanceid = ttbd.lineitem_resourceid + WHERE 1=1 + GROUP BY ttbd.lineitem_productcode + , DATE(ttbd.lineitem_usagestartdate) + , sm.service_cd + ON DUPLICATE KEY UPDATE total_cost = values(total_cost) + + + diff --git a/costCollector/src/main/resources/mapper/unused/Collect_SQL.xml b/costCollector/src/main/resources/mapper/unused/Collect_SQL.xml index f44f03c..0666133 100644 --- a/costCollector/src/main/resources/mapper/unused/Collect_SQL.xml +++ b/costCollector/src/main/resources/mapper/unused/Collect_SQL.xml @@ -8,23 +8,45 @@ - insert into unused_collector(csp_resourceid, create_dt) + insert into unused_collector(csp_resourceid, create_dt, instance_type, rsrc_type, region_id, region_nm, operation, instance_family_type, product_sku, instance_family) values (#{instanceid} - , #{createDt}) - ON DUPLICATE KEY UPDATE csp_resourceid = #{instanceid}, create_dt = #{createDt} + , #{createDt} + , #{instanceType} + , #{resourceType} + , #{regionId} + , #{regionNM} + , #{operation} + , #{instanceFamilyType} + , #{productSKU} + , #{instanceFamily}) + ON DUPLICATE KEY UPDATE instance_type = #{instanceType} + , rsrc_type = #{resourceType} + , region_id = #{regionId} + , region_nm = #{regionNM} + , operation = #{operation} + , instance_family_type = #{instanceFamilyType} + , product_sku = #{productSKU} + , instance_family = #{instanceFamily} diff --git a/costCollector/src/test/java/com/mcmp/collector/AwsTests.java b/costCollector/src/test/java/com/mcmp/collector/AwsTests.java new file mode 100644 index 0000000..ac64096 --- /dev/null +++ b/costCollector/src/test/java/com/mcmp/collector/AwsTests.java @@ -0,0 +1,28 @@ +package com.mcmp.collector; + +import com.mcmp.collector.service.Cur; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.time.LocalDateTime; + + +@RunWith(SpringRunner.class) +@SpringBootTest +@TestPropertySource(locations = "classpath:application.properties") +public class AwsTests { + + @Autowired + private Cur service; + + @Test + public void getS3List(){ + // service.batchInsertCURData("mcmpcostopti", LocalDateTime.now(), "1", "202408", null); + } + +} diff --git a/costProcessor/src/main/java/com/processor/costprocessor/batch/AbnormalProcess.java b/costProcessor/src/main/java/com/processor/costprocessor/batch/AbnormalProcess.java new file mode 100644 index 0000000..d6e9c52 --- /dev/null +++ b/costProcessor/src/main/java/com/processor/costprocessor/batch/AbnormalProcess.java @@ -0,0 +1,165 @@ +package com.processor.costprocessor.batch; + +import com.processor.costprocessor.model.abnormal.AbnormalItemModel; +import com.processor.costprocessor.model.unused.DailyAssetAmountModel; +import com.processor.costprocessor.model.util.AlarmReqModel; +import com.processor.costprocessor.util.AlarmService; +import com.processor.costprocessor.util.CalDatetime; +import com.processor.costprocessor.util.NumericCalculator; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.ibatis.session.SqlSessionFactory; +import org.mybatis.spring.batch.MyBatisBatchItemWriter; +import org.mybatis.spring.batch.MyBatisPagingItemReader; +import org.mybatis.spring.batch.builder.MyBatisBatchItemWriterBuilder; +import org.mybatis.spring.batch.builder.MyBatisPagingItemReaderBuilder; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.configuration.annotation.JobScope; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.core.step.skip.AlwaysSkipItemSkipPolicy; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.batch.item.ItemWriter; +import org.springframework.batch.item.support.CompositeItemWriter; +import org.springframework.batch.item.support.builder.CompositeItemWriterBuilder; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.transaction.PlatformTransactionManager; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +@Configuration +@RequiredArgsConstructor +public class AbnormalProcess { + + @Autowired + @Qualifier("sqlSessionBatch") + private SqlSessionFactory sqlSessionFactory; + + @Autowired + private CalDatetime calDatetime; + + @Autowired + private AlarmService alarmService; + + @Autowired + private NumericCalculator numericCalculator; + + @Bean + public CustomerItemListener customerListener(){ + return new CustomerItemListener(); + } + + + @Bean + public Job abnormalProcessJob(JobRepository jobRepository, Step abnormalProcessStep) { + return new JobBuilder("abnormalProcessJob", jobRepository) + .start(abnormalProcessStep) + .build(); + } + + @Bean + @JobScope + public Step abnormalProcessStep(JobRepository jobRepository, PlatformTransactionManager platformTransactionManager){ + return new StepBuilder("abnormalProcessStep", jobRepository) + .chunk(50, platformTransactionManager) + .reader(abnormalReader()) + .processor(abnormalProcessor()) + .writer(abnormalWriterItems()) + .startLimit(2) + .faultTolerant() + .retry(Exception.class) + .retryLimit(2) + .skip(Exception.class) + .skipPolicy(new AlwaysSkipItemSkipPolicy()) + .listener(customerListener()) + .build(); + } + + @Bean + @StepScope + public MyBatisPagingItemReader abnormalReader(){ + Map param = new HashMap<>(); + param.put("standardDT", calDatetime.getYesterDate().toString()); + param.put("subjectDTs", calDatetime.getLastMonthWeeklyDates().stream() + .map(LocalDate::toString) + .collect(Collectors.toList())); + + return new MyBatisPagingItemReaderBuilder() + .sqlSessionFactory(sqlSessionFactory) + .queryId("abnoraml.getAbnormalSubjectsCost") + .parameterValues(param) + .pageSize(50) + .build(); + } + + @Bean + @StepScope + public ItemProcessor abnormalProcessor(){ + LocalDate collect_dt = LocalDate.now(); + + return item -> { + double percent; + if(item.getSubject_cost() != 0){ + percent = ((item.getStandard_cost() - item.getSubject_cost()) / item.getSubject_cost()) * 100; + } else { + percent = 0; + } + item.setPercentage_point(percent); + + if(percent >= 30){ + item.setAbnormal_rating("Critical"); + } else if(percent >= 20){ + item.setAbnormal_rating("Caution"); + } else if(percent >= 10){ + item.setAbnormal_rating("Warning"); + } else { + return null; + } + + item.setCollect_dt(collect_dt); + String pctFormatter = String.format("%.2f", item.getPercentage_point()); + + AlarmReqModel alarmReqModel = AlarmReqModel.builder() + .event_type("Abnormal") + .resource_id(item.getProduct_cd()) + .resource_type(item.getProduct_cd()) + .csp_type(item.getCsp_type()) + .urgency("Warning") + .plan(item.getAbnormal_rating()) + .note("지난달 비용(" + numericCalculator.parseExponentialFormat(item.getSubject_cost()) + " USD) 대비 이번달 비용(" + numericCalculator.parseExponentialFormat(item.getStandard_cost()) + " USD)이 " + pctFormatter + " % 발생했습니다.") + .project_cd(item.getProject_cd()) + .build(); + + alarmService.sendAlarm(alarmReqModel); + + return item; + }; + } + + @Bean + public MyBatisBatchItemWriter abnormalWriterItems() { + return new MyBatisBatchItemWriterBuilder() + .sqlSessionFactory(sqlSessionFactory) + .statementId("abnoraml.insertDailyAbnoramlCost") + .build(); + } + + +} diff --git a/costProcessor/src/main/java/com/processor/costprocessor/batch/DelayItemWriter.java b/costProcessor/src/main/java/com/processor/costprocessor/batch/DelayItemWriter.java new file mode 100644 index 0000000..0c3fe8e --- /dev/null +++ b/costProcessor/src/main/java/com/processor/costprocessor/batch/DelayItemWriter.java @@ -0,0 +1,19 @@ +package com.processor.costprocessor.batch; + +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; + + +public class DelayItemWriter implements ItemWriter { + + private final long delayInMillis; + + public DelayItemWriter(long delayInMillis) { + this.delayInMillis = delayInMillis; + } + + @Override + public void write(Chunk items) throws Exception { + Thread.sleep(delayInMillis); + } +} diff --git a/costProcessor/src/main/java/com/processor/costprocessor/batch/UnusedProcess.java b/costProcessor/src/main/java/com/processor/costprocessor/batch/UnusedProcess.java index 965ef45..33eb021 100644 --- a/costProcessor/src/main/java/com/processor/costprocessor/batch/UnusedProcess.java +++ b/costProcessor/src/main/java/com/processor/costprocessor/batch/UnusedProcess.java @@ -126,7 +126,12 @@ public ItemProcessor> processor() List dailyAssetAmounts = unusedDao.getDailyAssetAmount(param); - return dailyAssetAmounts; + if(dailyAssetAmounts.isEmpty()){ + log.warn("Unused Daily Asset Amount INFO is EMPTY!"); + return null; + } else{ + return dailyAssetAmounts; + } }; } @@ -175,7 +180,7 @@ public ItemWriter> writerTask2() { Map body = new HashMap<>(); body.put("resource_id", itemList.get(0).getResource_id()); - body.put("create_dt", ZonedDateTime.now(ZoneId.of("UTC")).toLocalDateTime()); + body.put("create_dt", ZonedDateTime.now().toLocalDateTime()); HttpEntity httpEntity = new HttpEntity<>(body, httpHeaders); restTemplate.exchange(apiUrl, HttpMethod.POST, httpEntity, String.class); @@ -183,10 +188,6 @@ public ItemWriter> writerTask2() { String cleintErrorMsg = clientError.getMessage(); log.error("FAIL TO CALL COST SELECTOR - UNUSED API : " + cleintErrorMsg); } - - for(DailyAssetAmountModel item : itemList){ - log.info("Writer: 쓴다 " + item); - } } }; } diff --git a/costProcessor/src/main/java/com/processor/costprocessor/config/DatabaseConfig.java b/costProcessor/src/main/java/com/processor/costprocessor/config/DatabaseConfig.java index ad68ea8..43a33fb 100644 --- a/costProcessor/src/main/java/com/processor/costprocessor/config/DatabaseConfig.java +++ b/costProcessor/src/main/java/com/processor/costprocessor/config/DatabaseConfig.java @@ -38,7 +38,7 @@ public PlatformTransactionManager transactionManagerBatch() { public SqlSessionFactory sqlSessionBatch() throws Exception { SqlSessionFactoryBean sqlSession = new SqlSessionFactoryBean(); sqlSession.setDataSource(dataSourceBatch()); - sqlSession.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(ResourcePatternResolver.CLASSPATH_URL_PREFIX + "/mapper/unused/*_SQL.xml")); + sqlSession.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(ResourcePatternResolver.CLASSPATH_URL_PREFIX + "/mapper/**/*_SQL.xml")); return sqlSession.getObject(); } @@ -46,7 +46,7 @@ public SqlSessionFactory sqlSessionBatch() throws Exception { public SqlSessionFactory sqlSessionSimpleFactory() throws Exception { SqlSessionFactoryBean sqlSession = new SqlSessionFactoryBean(); sqlSession.setDataSource(dataSourceBatch()); - sqlSession.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(ResourcePatternResolver.CLASSPATH_URL_PREFIX + "/mapper/unused/*_SQL.xml")); + sqlSession.setMapperLocations(new PathMatchingResourcePatternResolver().getResources(ResourcePatternResolver.CLASSPATH_URL_PREFIX + "/mapper/**/*_SQL.xml")); return sqlSession.getObject(); } diff --git a/costProcessor/src/main/java/com/processor/costprocessor/model/abnormal/AbnormalItemModel.java b/costProcessor/src/main/java/com/processor/costprocessor/model/abnormal/AbnormalItemModel.java new file mode 100644 index 0000000..d7b5a7a --- /dev/null +++ b/costProcessor/src/main/java/com/processor/costprocessor/model/abnormal/AbnormalItemModel.java @@ -0,0 +1,18 @@ +package com.processor.costprocessor.model.abnormal; + +import lombok.Data; + +import java.time.LocalDate; + +@Data +public class AbnormalItemModel { + + private LocalDate collect_dt; + private String abnormal_rating; + private double percentage_point; + private String product_cd; + private double standard_cost; + private double subject_cost; + private String project_cd; + private String csp_type; +} diff --git a/costProcessor/src/main/java/com/processor/costprocessor/model/util/AlarmReqModel.java b/costProcessor/src/main/java/com/processor/costprocessor/model/util/AlarmReqModel.java new file mode 100644 index 0000000..679f540 --- /dev/null +++ b/costProcessor/src/main/java/com/processor/costprocessor/model/util/AlarmReqModel.java @@ -0,0 +1,20 @@ +package com.processor.costprocessor.model.util; + +import lombok.Builder; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@Builder +public class AlarmReqModel { + private String event_type; + private String resource_id; + private String resource_type; + private String csp_type; + private String account_id; + private String urgency; + private String plan; + private String note; + private String project_cd; +} diff --git a/costProcessor/src/main/java/com/processor/costprocessor/schedule/AbnormalJob.java b/costProcessor/src/main/java/com/processor/costprocessor/schedule/AbnormalJob.java new file mode 100644 index 0000000..cf32e9a --- /dev/null +++ b/costProcessor/src/main/java/com/processor/costprocessor/schedule/AbnormalJob.java @@ -0,0 +1,45 @@ +package com.processor.costprocessor.schedule; + + +import lombok.extern.slf4j.Slf4j; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.scheduling.quartz.QuartzJobBean; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class AbnormalJob extends QuartzJobBean { + + @Autowired + private JobLauncher jobLauncher; + + @Qualifier("abnormalProcessJob") + @Autowired + private Job abnormalProcessJob; + + @Override + protected void executeInternal(JobExecutionContext context) throws JobExecutionException { + log.info("Starting Abnormal Quartz Job"); + + try { + + JobParameters jobParameter = new JobParametersBuilder() + .addString("jobType", "Abnormal") + .addLong("createTime", System.currentTimeMillis()) + .toJobParameters(); + jobLauncher.run(abnormalProcessJob, jobParameter); + } catch (Exception e) { + log.error("[Error::Abnoraml] executing Spring Batch Job", e); + throw new JobExecutionException("Failed to execute Spring Batch Job", e); + } + + log.info("Quartz Abnoraml Job completed"); + } +} diff --git a/costProcessor/src/main/java/com/processor/costprocessor/schedule/ScheduleConfig.java b/costProcessor/src/main/java/com/processor/costprocessor/schedule/ScheduleConfig.java index f54ec8d..590e94c 100644 --- a/costProcessor/src/main/java/com/processor/costprocessor/schedule/ScheduleConfig.java +++ b/costProcessor/src/main/java/com/processor/costprocessor/schedule/ScheduleConfig.java @@ -11,6 +11,9 @@ public class ScheduleConfig { @Value("${unusedProcessCronSchedule}") public String batchCron; + @Value("${abnormalProcessCronSchedule}") + public String abnormalCron; + @Bean public JobDetail UnusedProcessJobDetail() { return JobBuilder.newJob().ofType(ScheduleJob.class) @@ -21,13 +24,33 @@ public JobDetail UnusedProcessJobDetail() { } @Bean - public Trigger JobTrigger(JobDetail UnusedJobDetail) { + public Trigger JobTrigger(JobDetail UnusedProcessJobDetail) { CronScheduleBuilder cronSchedule = CronScheduleBuilder.cronSchedule(batchCron); - return TriggerBuilder.newTrigger().forJob(UnusedJobDetail) + return TriggerBuilder.newTrigger().forJob(UnusedProcessJobDetail) .withIdentity("UnusedProcessJobTrigger1", "Unused") .withDescription("Unused Job Trigger") .withSchedule(cronSchedule) .build(); } + + @Bean + public JobDetail AbnormalProcessJobDetail() { + return JobBuilder.newJob().ofType(AbnormalJob.class) + .storeDurably() + .withIdentity("AbnoramlProcessJob", "Abnormal") + .withDescription("Execute Spring Batch Job with Quartz") + .build(); + } + + @Bean + public Trigger AbnormalJobTrigger(JobDetail AbnormalProcessJobDetail) { + CronScheduleBuilder cronSchedule = CronScheduleBuilder.cronSchedule(abnormalCron); + + return TriggerBuilder.newTrigger().forJob(AbnormalProcessJobDetail) + .withIdentity("AbnormalProcessJobTrigger1", "Abnormal") + .withDescription("Abnormal Job Trigger") + .withSchedule(cronSchedule) + .build(); + } } diff --git a/costProcessor/src/main/java/com/processor/costprocessor/util/AlarmService.java b/costProcessor/src/main/java/com/processor/costprocessor/util/AlarmService.java new file mode 100644 index 0000000..69d2c32 --- /dev/null +++ b/costProcessor/src/main/java/com/processor/costprocessor/util/AlarmService.java @@ -0,0 +1,55 @@ +package com.processor.costprocessor.util; + +import com.processor.costprocessor.model.util.AlarmReqModel; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.RestTemplate; + +import java.time.ZonedDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +@Slf4j +public class AlarmService { + + @Value("${opti.alarm.url}") + private String alarmURL; + + public void sendAlarm(AlarmReqModel req){ + try{ + String apiUrl = String.format("%s/alert/sendOptiAlarmMail", alarmURL); + RestTemplate restTemplate = new RestTemplate(); + + HttpHeaders httpHeaders = new HttpHeaders(); + httpHeaders.setContentType(MediaType.APPLICATION_JSON); + + Map body = new HashMap<>(); + body.put("alarm_type", List.of("mail", "slack")); + body.put("event_type", req.getEvent_type()); + body.put("resource_id", req.getResource_id()); + body.put("resource_type", req.getResource_type()); + body.put("csp_type", req.getCsp_type()); + body.put("account_id", req.getAccount_id()); + body.put("urgency", req.getUrgency()); + body.put("plan", req.getPlan()); + body.put("note", req.getNote()); + body.put("project_cd", req.getProject_cd()); + HttpEntity httpEntity = new HttpEntity<>(body, httpHeaders); + + restTemplate.exchange(apiUrl, HttpMethod.POST, httpEntity, String.class); + } catch (HttpClientErrorException | HttpServerErrorException clientError) { + String cleintErrorMsg = clientError.getMessage(); + log.error("FAIL TO SEND OPTI ALARM : " + cleintErrorMsg); + } + } + +} diff --git a/costProcessor/src/main/java/com/processor/costprocessor/util/CalDatetime.java b/costProcessor/src/main/java/com/processor/costprocessor/util/CalDatetime.java index 93d0302..2964a8e 100644 --- a/costProcessor/src/main/java/com/processor/costprocessor/util/CalDatetime.java +++ b/costProcessor/src/main/java/com/processor/costprocessor/util/CalDatetime.java @@ -3,14 +3,15 @@ import com.processor.costprocessor.model.util.CurrentDateModel; import org.springframework.stereotype.Service; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZonedDateTime; +import java.time.*; +import java.time.temporal.TemporalAdjusters; +import java.util.ArrayList; +import java.util.List; @Service public class CalDatetime { public CurrentDateModel currentDatetime(){ - ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC")); + ZonedDateTime now = ZonedDateTime.now(); LocalDateTime startOfToday = now.toLocalDate().atStartOfDay(); LocalDateTime endOfToday = now.toLocalDate().plusDays(1).atStartOfDay(); @@ -22,9 +23,9 @@ public CurrentDateModel currentDatetime(){ } public CurrentDateModel monthDateRange(){ - ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC")); + ZonedDateTime now = ZonedDateTime.now(); LocalDateTime startOfToday = now.toLocalDate().minusMonths(1).atStartOfDay(); - LocalDateTime endOfToday = now.toLocalDateTime(); + LocalDateTime endOfToday = now.toLocalDate().atStartOfDay(); CurrentDateModel curDt = new CurrentDateModel(); curDt.setCurStartDatetime(startOfToday); @@ -32,4 +33,28 @@ public CurrentDateModel monthDateRange(){ return curDt; } + + + public LocalDate getYesterDate(){ + return LocalDate.now().minusDays(1); + } + + public List getLastMonthWeeklyDates(){ + List result = new ArrayList<>(); + + LocalDate yesterday = LocalDate.now().minusDays(1); + LocalDate lastMonthSameDate = yesterday.minusMonths(1); + DayOfWeek dayOfWeek = yesterday.getDayOfWeek(); + LocalDate lastMonthFirstDay = lastMonthSameDate.with(TemporalAdjusters.firstDayOfMonth()); + + for(LocalDate date = lastMonthFirstDay; + !date.isAfter(lastMonthSameDate.with(TemporalAdjusters.lastDayOfMonth())); + date = date.plusDays(1)){ + if(date.getDayOfWeek() == dayOfWeek){ + result.add(date); + } + } + + return result; + } } diff --git a/costProcessor/src/main/java/com/processor/costprocessor/util/NumericCalculator.java b/costProcessor/src/main/java/com/processor/costprocessor/util/NumericCalculator.java new file mode 100644 index 0000000..420f815 --- /dev/null +++ b/costProcessor/src/main/java/com/processor/costprocessor/util/NumericCalculator.java @@ -0,0 +1,27 @@ +package com.processor.costprocessor.util; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.text.DecimalFormat; + +@Service +@Slf4j +public class NumericCalculator { + + public String parseExponentialFormat(double num){ + String numStr = Double.toString(num); + + if(numStr.contains("E")){ + DecimalFormat df = new DecimalFormat("0.00E0"); + String formatted = df.format(num); + + formatted = formatted.replace("E", "(10^") + ")"; + + return formatted; + } else { + return numStr; + } + + } +} diff --git a/costProcessor/src/main/resources/application.properties b/costProcessor/src/main/resources/application.properties index 9c7bad6..1cbb98c 100644 --- a/costProcessor/src/main/resources/application.properties +++ b/costProcessor/src/main/resources/application.properties @@ -12,9 +12,12 @@ spring.datasource.hikari.batch.max-wait-millis=30000 spring.datasource.hikari.batch.remove-abandoned-on-borrow=true spring.datasource.hikari.batch.remove-abandoned-timeout=30 +spring.batch.job.enabled=false unusedProcessCronSchedule=0 45 * * * ? +abnormalProcessCronSchedule=0 0 1,7 * * ? logging.level.org.springframework.batch=DEBUG logging.level.org.mybatis=DEBUG cost.selector.url=http://localhost:8083 +opti.alarm.url=http://localhost:9000 diff --git a/costProcessor/src/main/resources/mapper/abnormal/Abnormal_SQL.xml b/costProcessor/src/main/resources/mapper/abnormal/Abnormal_SQL.xml new file mode 100644 index 0000000..8bb1553 --- /dev/null +++ b/costProcessor/src/main/resources/mapper/abnormal/Abnormal_SQL.xml @@ -0,0 +1,45 @@ + + + + + + + INSERT INTO daily_abnormal_by_product(collect_dt, product_cd, abnormal_rating, percentage_point, standard_cost, subject_cost, project_cd, csp_type) + VALUES (#{collect_dt} + , #{product_cd} + , #{abnormal_rating} + , #{percentage_point} + , #{standard_cost} + , #{subject_cost} + , #{project_cd} + , #{csp_type}) + ON DUPLICATE KEY UPDATE abnormal_rating = #{abnormal_rating} + , percentage_point = #{percentage_point} + , standard_cost = #{standard_cost} + , subject_cost = #{subject_cost} + + diff --git a/costProcessor/src/main/resources/mapper/unused/AWS_SQL.xml b/costProcessor/src/main/resources/mapper/unused/AWS_SQL.xml index fb6d861..80132eb 100644 --- a/costProcessor/src/main/resources/mapper/unused/AWS_SQL.xml +++ b/costProcessor/src/main/resources/mapper/unused/AWS_SQL.xml @@ -12,7 +12,7 @@ @@ -30,7 +42,8 @@ SELECT CASE WHEN THEN 'TRUE' ELSE 'FALSE' - END AS max_amount + END AS max_amount_yn + , MAX(upm.metric_avg_amount) AS max_amount , AVG(upm.metric_avg_amount) AS avg_amount , COUNT(*) AS c_total_count FROM unused_process_mart upm @@ -44,6 +57,7 @@ SELECT COUNT(*) AS n_total_count , COUNT(CASE WHEN THEN 1 END) AS count_less_than_setting , COUNT(CASE WHEN 0 ]]> THEN 1 END) AS count_occured + , AVG(upm.metric_avg_amount) AS avg_amount FROM unused_process_mart upm WHERE 1=1 AND upm.resource_id = #{resource_id} @@ -61,4 +75,126 @@ ON DUPLICATE KEY UPDATE plan_type = #{plan_type} + + + + + + INSERT INTO inst_opti_rcmd_rst(create_dt, resource_id, csp_type, csp_account, origin_type, rcmd_type, plan_type, origin_usd, rcmd_usd) + VALUES (#{createDT} + , #{resourceID} + , #{cspType} + , #{cspAccount} + , #{originType} + , #{rcmdType} + , #{planType} + , #{originUSD} + , #{rcmdUSD}) + ON DUPLICATE KEY UPDATE origin_type = #{originType} + , rcmd_type = #{rcmdType} + , plan_type = #{planType} + , origin_usd = #{originUSD} + , rcmd_usd = #{rcmdUSD} + + diff --git a/docker-compose.yml b/docker-compose.yml index 151a585..83ff227 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,12 @@ services: spring.datasource.hikari.batch.jdbc-url: ${COST_DB_URL} spring.datasource.hikari.batch.username: ${COST_DB_USERNM} spring.datasource.hikari.batch.password: ${COST_DB_PW} - unusedBatchCronSchedule: ${COST_COLLECT_CRON_SCHEDULE} + unusedBatchCronSchedule: ${COST_COLLECT_UNUSED_CRON_SCHEDULE} + curBatchCronSchedule: ${COST_COLLECT_CUR_CRON_SCHEDULE} + aws.data.export.name: ${AWS_CUR_EXPORT_NAME} + aws.data.export.path.prefix: ${AWS_CUR_EXPORT_PATH_PREFIX} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} costprocessor: restart: always build: @@ -45,8 +50,10 @@ services: spring.datasource.hikari.batch.jdbc-url: ${COST_DB_URL} spring.datasource.hikari.batch.username: ${COST_DB_USERNM} spring.datasource.hikari.batch.password: ${COST_DB_PW} - unusedProcessCronSchedule: ${COST_PROCESS_CRON_SCHEDULE} + unusedProcessCronSchedule: ${COST_PROCESS_UNUSED_CRON_SCHEDULE} + abnormalProcessCronSchedule: ${COST_PROCESS_ABNORMAL_CRON_SCHEDULE} cost.selector.url: ${COST_SELECTOR_URL} + opti.alarm.url: ${ALARM_URL} costselector: restart: always build: @@ -59,6 +66,7 @@ services: spring.datasource.hikari.cost.optimize.jdbc-url: ${COST_DB_URL} spring.datasource.hikari.cost.optimize.username: ${COST_DB_USERNM} spring.datasource.hikari.cost.optimize.password: ${COST_DB_PW} + opti.alarm.url: ${ALARM_URL} alarmservice: restart: always build: @@ -77,6 +85,20 @@ services: spring.datasource.hikari.history.jdbc-url: ${COST_DB_URL} spring.datasource.hikari.history.username: ${COST_DB_USERNM} spring.datasource.hikari.history.password: ${COST_DB_PW} + assetcollector: + restart: always + build: + context: ./assetCollector + networks: + - mcmp_cost_network + ports: + - "8091:8091" + environment: + spring.datasource.hikari.batch.jdbc-url: ${COST_DB_URL} + spring.datasource.hikari.batch.username: ${COST_DB_USERNM} + spring.datasource.hikari.batch.password: ${COST_DB_PW} + asset.collect.url: ${ASSET_MONITORING_SERVER} + assetCollectBatchCronSchedule: ${ASSET_COLLECT_BATCH_CRON_SCHEDULE} networks: mcmp_cost_network: driver: bridge diff --git a/mysql/init_cost_db_ddl.sql b/mysql/init_cost_db_ddl.sql new file mode 100644 index 0000000..8589d1e --- /dev/null +++ b/mysql/init_cost_db_ddl.sql @@ -0,0 +1,260 @@ +CREATE DATABASE IF NOT EXISTS cost; + +USE cost; + +-- cost.alarm_history definition + +CREATE TABLE `alarm_history` ( + `event_type` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `resource_id` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `resource_type` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `occure_dt` timestamp NOT NULL, + `account_id` varchar(100) DEFAULT NULL, + `urgency` varchar(20) DEFAULT NULL, + `plan` varchar(20) DEFAULT NULL, + `note` varchar(200) DEFAULT NULL, + `occure_date` timestamp NOT NULL, + `csp_type` varchar(10) NOT NULL, + `alarm_impl` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `project_cd` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'BLANK_00_', + PRIMARY KEY (`event_type`,`resource_id`,`resource_type`,`occure_date`,`csp_type`,`alarm_impl`,`project_cd`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.asset_compute_metric definition + +CREATE TABLE `asset_compute_metric` ( + `csp_type` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `csp_account` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `csp_instanceid` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `collect_dt` timestamp NOT NULL, + `metric_type` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT 'CPU/NETWORK', + `metric_amount` double DEFAULT NULL, + `resource_status` varchar(100) DEFAULT NULL, + `resource_spot_yn` varchar(1) DEFAULT 'N', + PRIMARY KEY (`csp_type`,`csp_instanceid`,`collect_dt`,`metric_type`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.asset_rsopt_settings definition + +CREATE TABLE `asset_rsopt_settings` ( + `csp_type` varchar(100) DEFAULT NULL, + `metric_type` varchar(100) DEFAULT NULL, + `regress_duration` int DEFAULT NULL, + `criteria_value` double DEFAULT NULL, + `cmp_user_id` varchar(100) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.cur_origin definition + +CREATE TABLE `cur_origin` ( + `lineitem_usageaccountid` varchar(100) NOT NULL COMMENT 'AWS 서브계정 구분 ID', + `lineitem_productcode` varchar(100) NOT NULL COMMENT '서비스구분코드', + `lineitem_resourceid` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL COMMENT '인스턴스 구분 코드', + `lineitem_lineitemtype` varchar(100) DEFAULT NULL COMMENT '비용구분', + `product_instancetype` varchar(100) DEFAULT NULL COMMENT '서비스 상세 구분', + `pricing_unit` varchar(100) DEFAULT NULL COMMENT '사용량 단위', + `lineitem_usageamount` varchar(100) DEFAULT NULL COMMENT '사용량', + `lineitem_unblendedcost` varchar(100) DEFAULT NULL COMMENT 'on-demand 비용', + `lineitem_blendedcost` varchar(100) DEFAULT NULL COMMENT '할인 청구 비용', + `lineitem_usagestartdate` timestamp NULL DEFAULT NULL COMMENT '인스턴스 사용 시작 일시', + `lineitem_usageenddate` timestamp NULL DEFAULT NULL COMMENT '인스턴스 사용 끝 일시', + `pricing_publicondemandcost` varchar(100) DEFAULT NULL COMMENT 'on-demand 총비용', + `pricing_publicondemandrate` varchar(100) DEFAULT NULL COMMENT 'on-demand 단가', + `lineitem_currencycode` varchar(20) DEFAULT NULL COMMENT '화폐단위', + `data_collect_date` timestamp NULL DEFAULT NULL, + `data_collect_seq` varchar(100) DEFAULT NULL, + `product_sku` varchar(100) DEFAULT NULL, + `product_region` varchar(50) DEFAULT NULL, + `product_instanceFamily` varchar(100) DEFAULT NULL, + `product_location` varchar(100) DEFAULT NULL, + `lineitem_operation` varchar(100) DEFAULT NULL, + `product_instancetypefamily` varchar(20) DEFAULT NULL, + `lineitem_usagetype` varchar(100) DEFAULT NULL, + `product_vcpu` varchar(30) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `product_memory` varchar(30) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.cur_process_info definition + +CREATE TABLE `cur_process_info` ( + `csp` varchar(10) NOT NULL, + `payer_account` varchar(100) NOT NULL, + `collect_date` varchar(10) NOT NULL, + `object_key` varchar(200) DEFAULT NULL, + `certifed_fixed_yn` varchar(1) DEFAULT NULL, + `certifed_fixed_date` timestamp NULL DEFAULT NULL, + PRIMARY KEY (`csp`,`payer_account`,`collect_date`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.daily_abnormal_by_product definition + +CREATE TABLE `daily_abnormal_by_product` ( + `collect_dt` date NOT NULL, + `product_cd` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `abnormal_rating` varchar(100) DEFAULT NULL, + `percentage_point` varchar(100) DEFAULT NULL, + `standard_cost` varchar(100) DEFAULT NULL, + `subject_cost` varchar(100) DEFAULT NULL, + `project_cd` varchar(100) NOT NULL, + `workspace_cd` varchar(100) DEFAULT NULL, + `csp_type` varchar(10) DEFAULT NULL, + PRIMARY KEY (`collect_dt`,`product_cd`,`project_cd`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.daily_summation_by_product definition + +CREATE TABLE `daily_summation_by_product` ( + `date` date NOT NULL, + `product` varchar(100) NOT NULL, + `total_cost` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `project_cd` varchar(100) NOT NULL, + `workspace_cd` varchar(100) DEFAULT NULL, + `csp_type` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + PRIMARY KEY (`date`,`product`,`project_cd`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.inst_opti_rcmd_rst definition + +CREATE TABLE `inst_opti_rcmd_rst` ( + `create_dt` timestamp NOT NULL, + `resource_id` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `csp_type` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `csp_account` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `origin_type` varchar(100) DEFAULT NULL, + `rcmd_type` varchar(100) DEFAULT NULL, + `plan_type` varchar(20) DEFAULT NULL, + `origin_usd` varchar(100) DEFAULT NULL, + `rcmd_usd` varchar(100) DEFAULT NULL, + PRIMARY KEY (`create_dt`,`resource_id`,`csp_type`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.monthly_summation definition + +CREATE TABLE `monthly_summation` ( + `year_month` varchar(6) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `project_cd` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `csp` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `total_cost` varchar(100) DEFAULT NULL, + `workspace_cd` varchar(100) DEFAULT NULL, + PRIMARY KEY (`year_month`,`project_cd`,`csp`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.service_category definition + +CREATE TABLE `service_category` ( + `service_cd` varchar(100) NOT NULL COMMENT '서비스 코드', + `service_nm` varchar(100) NOT NULL COMMENT '서비스 명', + `service_type` varchar(100) NOT NULL COMMENT '서비스 타입', + PRIMARY KEY (`service_cd`,`service_nm`,`service_type`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.servicegroup_meta definition + +CREATE TABLE `servicegroup_meta` ( + `csp_type` varchar(100) NOT NULL COMMENT 'CSP 종류', + `csp_account` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT 'mcmpcostopti' COMMENT 'CSP 계정 ID', + `csp_instanceid` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'common' COMMENT ' 인스턴스 구분코드', + `service_cd` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '서비스 코드', + `service_nm` varchar(100) DEFAULT NULL COMMENT '서비스 명', + `service_type` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT 'project' COMMENT '서비스 타입', + `workspace_cd` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT 'testWs' COMMENT 'WorkSpace 코드', + `service_uid` varchar(100) DEFAULT NULL, + `vm_id` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'testPrj', + `vm_uid` varchar(100) DEFAULT NULL, + `vm_nm` varchar(100) DEFAULT NULL, + `mci_id` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT 'testPrj', + `mci_uid` varchar(100) DEFAULT NULL, + `mci_nm` varchar(100) DEFAULT NULL, + `instance_running_status` varchar(1) DEFAULT 'Y', + PRIMARY KEY (`csp_type`,`csp_instanceid`,`service_cd`,`vm_id`,`mci_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.tbl_table_billing_detail_202409 definition + +CREATE TABLE `tbl_table_billing_detail_202409` ( + `lineitem_usageaccountid` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_productcode` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_resourceid` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_lineitemtype` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `product_instancetype` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `pricing_unit` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_usageamount` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_unblendedcost` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_blendedcost` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_usagestartdate` timestamp NULL DEFAULT NULL, + `lineitem_usageenddate` timestamp NULL DEFAULT NULL, + `pricing_publicondemandcost` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `pricing_publicondemandrate` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `lineitem_currencycode` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `product_sku` varchar(100) DEFAULT NULL, + `product_region` varchar(50) DEFAULT NULL, + `product_instanceFamily` varchar(100) DEFAULT NULL, + `product_location` varchar(100) DEFAULT NULL, + `lineitem_operation` varchar(100) DEFAULT NULL, + `product_instancetypefamily` varchar(20) DEFAULT NULL, + `lineitem_usagetype` varchar(100) DEFAULT NULL, + `product_vcpu` varchar(30) DEFAULT NULL, + `product_memory` varchar(30) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.temp_cmp_user_info definition + +CREATE TABLE `temp_cmp_user_info` ( + `mcmp_user_id` varchar(100) DEFAULT NULL, + `csp_type` varchar(10) DEFAULT NULL, + `csp_account_id` varchar(100) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.temp_cmp_user_mail_receiver definition + +CREATE TABLE `temp_cmp_user_mail_receiver` ( + `mcmp_user_id` varchar(100) DEFAULT NULL, + `mcmp_mail_receiver` varchar(100) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.temp_cmp_user_role_arn definition + +CREATE TABLE `temp_cmp_user_role_arn` ( + `mcmp_user_id` varchar(100) DEFAULT NULL, + `csp` varchar(20) DEFAULT NULL, + `bucket_nm` varchar(100) DEFAULT NULL, + `role_arn` varchar(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.unused_batch_rst definition + +CREATE TABLE `unused_batch_rst` ( + `create_dt` timestamp NOT NULL, + `csp_type` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `csp_account` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `csp_instanceid` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `plan_type` varchar(100) DEFAULT NULL, + PRIMARY KEY (`create_dt`,`csp_type`,`csp_account`,`csp_instanceid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.unused_collector definition + +CREATE TABLE `unused_collector` ( + `csp_resourceid` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, + `create_dt` timestamp NOT NULL, + `instance_type` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `rsrc_type` varchar(10) DEFAULT NULL, + `region_id` varchar(50) DEFAULT NULL, + `region_nm` varchar(100) DEFAULT NULL, + `operation` varchar(100) DEFAULT NULL, + `instance_family_type` varchar(20) DEFAULT NULL, + `product_sku` varchar(100) DEFAULT NULL, + `instance_family` varchar(100) DEFAULT NULL, + PRIMARY KEY (`csp_resourceid`,`create_dt`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +-- cost.unused_process_mart definition + +CREATE TABLE `unused_process_mart` ( + `create_dt` timestamp NOT NULL, + `resource_id` varchar(200) NOT NULL, + `collect_dt` timestamp NOT NULL, + `metric_type` varchar(100) NOT NULL, + `metric_avg_amount` double DEFAULT NULL, + PRIMARY KEY (`resource_id`,`collect_dt`,`metric_type`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;