From 335cdd4b32a52c1288ac6df4be42af19256c84d6 Mon Sep 17 00:00:00 2001 From: acialini Date: Mon, 14 Oct 2024 11:49:20 +0200 Subject: [PATCH 01/55] [PPANTT-137] feat: updated modules and workload identity for aks-leonardo and payopts domain --- src/aks-leonardo/.terraform.lock.hcl | 224 ++++++++---------- .../paymentoptions-app/.terraform.lock.hcl | 102 -------- .../paymentoptions-app/02_namespace.tf | 28 ++- .../03_serviceaccounts_azure_devops.tf | 2 +- .../04_apim_payment_options.tf | 2 +- .../04_apim_payment_options_mock.tf | 4 +- .../05_aks_middleware_tools.tf | 65 ++--- src/domains/paymentoptions-app/99_main.tf | 15 +- .../paymentoptions-common/.terraform.lock.hcl | 65 ----- .../paymentoptions-common/03_eventhub.tf | 4 +- .../10_github_identity.tf | 6 +- src/domains/paymentoptions-common/99_main.tf | 10 +- .../.terraform.lock.hcl | 107 --------- .../paymentoptions-secrets/01_keyvault.tf | 4 +- src/domains/paymentoptions-secrets/99_main.tf | 14 +- 15 files changed, 188 insertions(+), 464 deletions(-) delete mode 100644 src/domains/paymentoptions-app/.terraform.lock.hcl delete mode 100644 src/domains/paymentoptions-common/.terraform.lock.hcl delete mode 100644 src/domains/paymentoptions-secrets/.terraform.lock.hcl diff --git a/src/aks-leonardo/.terraform.lock.hcl b/src/aks-leonardo/.terraform.lock.hcl index e4489ead2b..5fab8d9bde 100644 --- a/src/aks-leonardo/.terraform.lock.hcl +++ b/src/aks-leonardo/.terraform.lock.hcl @@ -2,167 +2,143 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/alekc/kubectl" { - version = "2.0.4" + version = "2.1.0" constraints = "~> 2.0" hashes = [ - "h1:TUeUq1UdVkHTxcgq7CJWWXBrc8VEQTufmgU18qDmfGE=", - "zh:15c227886bac78c8b8827f85595648212574ec81febc39e1055e1a6bf048fe65", - "zh:2211ebeeb0918dbb3587d206e32adca9e1f343a93bbffcd37d8d99bf4d8dea9a", - "zh:2303836cdea12ece8dbe39c2d7d30a9378fd06e9c2ebda66cbe5e01cc096ee2e", - "zh:3687f69e531c70845682b214888a9959b93f2be3c2531801228a4b1965d59921", - "zh:4dd686b4c55e2eedd80464984c9bb736c2df7a96d9dd59a692d91d09173f5f64", - "zh:51e29c13a87e56867b4be0b0c68da874149bf6d4014d7259b62d91162142c1bd", - "zh:5d9d99260f2adfb8867068a3d7644336d57cfa7710062c5221dcbb5a7ec90c7d", - "zh:901c19d73da6688437b19a85e3cd60e8f2090c84699e108b31953bb87f6d3141", - "zh:9547743606a36fa6b6748c5e2e1959b6f185730a1da53a3c351cfa0d8c096687", - "zh:9772a30704e69b54de5a332858a39591f52286121cffcba702346830b1c6e362", - "zh:b44792f99d7c90b9a364dd922f861e459ae1b1edc039f6b3078549021fec4511", - "zh:b5eb871ed2e39b9236dce06170b1fd5dda29f3c1d53f8e08285ccb9a4f574201", - "zh:e8bb4c3d9f680977b560e9dec24662650f790259b2c1311ee07a72157f6492b3", - "zh:f4772cfa0f9c73fdef008bb917cd268620009dc7ff270a4d819125c642b5acce", + "h1:fyE+ICPznpHaRAIT/GtIUdl7Z5MqBpXhnLH26+FlpT8=", + "zh:030d9aaaa251fb9f2b98640f343b1944a09924a3507340590552f5dfb037c1e2", + "zh:1a1672cd6a60d0a5296bd89d92b2113af9105ce933629c0195416013744db16f", + "zh:1cfc7bfbe6f145acd08fb52289f0fe4ed36f3a5e0d93f6b221e40236d164a5b2", + "zh:36e2620433b497f1538d84647e7041042bc43de9b3491febe5cb9ec0b47401b8", + "zh:5b301ff79f6b80869d6f5e54abdc63d7dde146af9b3c37340f7af922321cf316", + "zh:6f63ce78866dc3f5ea127825a70a11d53cb93f5dfa6187e8390592dd2f8857f9", + "zh:73e51fe86ec9263ab60507b3c811875074532613abf73154ab848fda181e078a", + "zh:8e65fe5b8465f25fadb4a7411981aeb307e2f482060b2642795fe371883efbb2", + "zh:91c07d9120687ba93f13af24f44cdf19d0c96429da90b384d10c4bf2bcf5725e", + "zh:c53cdefc0a25113e09bdf3c57a1c064d937b783fbcf9bb9228e9309d95294b9e", + "zh:c652849feab85900c881af20effaa26052bdadba5eaafefce9d09e15c8c6c32f", + "zh:c739f54428c0ad83f7031ae29d56c377026619912b814ba03ad37e92df558125", + "zh:d0cd843e29984889be06a61e0eebe6dccf669563f8130d1066f50552507db66f", + "zh:e9eb47fdda142d1f51cdd486ff46bf089a9c55ec93ac1c6d36d2e757ed217ee5", ] } provider "registry.terraform.io/hashicorp/azuread" { - version = "2.47.0" - constraints = "<= 2.47.0" + version = "3.0.2" + constraints = "<= 3.0.2" hashes = [ - "h1:8J74v92UvtqVNucugAtB+Sd44oTgnhfct+Xf8ObOZug=", - "h1:KB9BNRNStbdsfdRmVXUwXtN77qgX5VjBy2UALcqp218=", - "h1:g8+gBFM4QVOEQFqAEs5pR6iXpbGvgPvcEi1evHwziyw=", - "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", - "h1:zYMGokLn44KSWir7Nr4t8lEAPMB6JuXd2LlP2Ac2tMY=", - "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", + "h1:HNrx7UJEDY5Kbx/r1LRQDWnziqvB6x3IU+pEA8Vq7dw=", + "zh:16e724b80a9004c7978c30f69a73c98ff63eb8a03937dd44c2a8f0ea0438b7a3", "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", - "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", - "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", - "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", - "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", - "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", - "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", - "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", - "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", - "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", - "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", + "zh:2bbbf13713ca4767267b889471c9fc14a56a8fdf5d1013da3ca78667e3caec64", + "zh:409ccb05431d643a079da082d89db2d95d6afed4769997ac537c8b7de3bff867", + "zh:53e4bca0f5d015380f7f524f36344afe6211ccaf614bfc69af73ca64a9f47d6c", + "zh:5780be2c1981d090604d7fa4cef675462f17f40e7f3dc501a031488e87a35b8f", + "zh:850e61a1b3e64c752c418526ccf48653514c861b36f5feb631619f906f7e99a0", + "zh:8c3565bfcea006a734149cc080452a9daf7d2a9d5362eb7e0a088b6c0d7f0f03", + "zh:908b9e6ad49d5d21173ecefc7924902047611be93bbf8e7d021aa9563358396f", + "zh:a2a79765c029bc58966eff61cb6e9b0ee14d2ac52b0a22fc7dfa35c9a49af669", + "zh:c7f56cbe8743e9ba81fce871bc97d9c07abe86770d9ee7ffefbf3882a61ba89a", + "zh:d4dba80e33421b30d81c62611fb7fc62ad39afecc6484436e635913cd8553e67", ] } provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.105.0" - constraints = "~> 3.30, <= 3.105.0" + version = "3.116.0" + constraints = "~> 3.30, ~> 3.100, <= 3.116.0" hashes = [ - "h1:MK83TecMdabDD+HjbxdTt3emXp8G6djLj7KvvUGstM0=", - "h1:OtWRTAMNOruOmwVB72QSGXC5IIGGQcHwEqnCCmsGbGM=", - "h1:SOC7EdvKd5YowghQvb6hu209F1PQqtb8LulbQkxOZQQ=", - "h1:tEDW5rEALglcH1JRy31z6AzDULECYrAZOD24B4mqry8=", - "h1:zWkzhP2fx0WQIAUp6Amk/We3WNcbtiWagpKF5PJP5+M=", - "zh:2f81bca6a3bf3d37604bf99fdb2c77d6118520aa379ab65fd28e6b76bed399cd", - "zh:3578eb79d175af9544b0dc543124d551c0fed4c48f51773ee17e1dc62e22833a", - "zh:377dbb56caea3fa1e6a6599193b55c8594204c40c054fc2ace4f576fdfe750a2", - "zh:3d1cf01929cb213ff9a0f9753e35699bf13f60f7f0f15b38f1b216fa2cbb5f72", - "zh:481376d79224a0e4aebc6e39dee10de3cc14efd1c7c58b6d74c538e356cf4bb2", - "zh:625119aec0d24ff693c589d802b7983ffce3fcf1e9c3351396af02799dd176ca", - "zh:65981e62a6e9eb8a39dd332632617e8c929dcce6af48d3829f590f5c0f14362f", - "zh:72db82697f4e694c21defa8d0efb22f71d373676d078d71d567e8b4d9a134df7", - "zh:a0fa43cf78716ff98eccd7506b017c5c487034d9d9cce88c1accdba9314a4822", - "zh:b073f60b68b0102128815251dd895ec7f24bddec84a1b725fc0777de8a78dc7e", - "zh:b601e509eb9735756b6b7ccacc15d6333769a7bb2f8ac8c394e39b29cfc6ee55", + "h1:2QbjtN4oMXzdA++Nvrj/wSmWZTPgXKOSFGGQCLEMrb4=", + "zh:02b6606aff025fc2a962b3e568e000300abe959adac987183c24dac8eb057f4d", + "zh:2a23a8ce24ff9e885925ffee0c3ea7eadba7a702541d05869275778aa47bdea7", + "zh:57d10746384baeca4d5c56e88872727cdc150f437b8c5e14f0542127f7475e24", + "zh:59e3ebde1a2e1e094c671e179f231ead60684390dbf02d2b1b7fe67a228daa1a", + "zh:5f1f5c7d09efa2ee8ddf21bd9efbbf8286f6e90047556bef305c062fa0ac5880", + "zh:a40646aee3c9907276dab926e6123a8d70b1e56174836d4c59a9992034f88d70", + "zh:c21d40461bc5836cf56ad3d93d2fc47f61138574a55e972ad5ff1cb73bab66dc", + "zh:c56fb91a5ae66153ba0f737a26da1b3d4f88fdef7d41c63e06c5772d93b26953", + "zh:d1e60e85f51d12fc150aeab8e31d3f18f859c32f927f99deb5b74cb1e10087aa", + "zh:ed35e727e7d79e687cd3d148f52b442961ede286e7c5b4da1dcd9f0128009466", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f6d2a4e7c58f44e7d04a4a9c73f35ed452f412c97c85def68c4b52814cbe03ab", ] } provider "registry.terraform.io/hashicorp/external" { - version = "2.3.3" - constraints = "<= 2.3.3" + version = "2.3.4" + constraints = "<= 2.3.4" hashes = [ - "h1:/x65slrvO8YG5MKxE2DaU5udEbUxBu3BgEiO7EEM9bQ=", - "h1:H+3QlVPs/7CDa3I4KU/a23wYeGeJxeBlgvR7bfK1t1w=", - "h1:Qi72kOSrEYgEt5itloFhDfmiFZ7wnRy3+F74XsRuUOw=", - "h1:Up2xaIhiNYomK8Lhe29U2FcojpbRWZYDtSeS03OhI94=", - "h1:gShzO1rJtADK9tDZMvMgjciVAzsBh39LNjtThCwX1Hg=", - "zh:03d81462f9578ec91ce8e26f887e34151eda0e100f57e9772dbea86363588239", - "zh:37ec2a20f6a3ec3a0fd95d3f3de26da6cb9534b30488bc45723e118a0911c0d8", - "zh:4eb5b119179539f2749ce9de0e1b9629d025990f062f4f4dddc161562bb89d37", - "zh:5a31bb58414f41bee5e09b939012df5b88654120b0238a89dfd6691ba197619a", - "zh:6221a05e52a6a2d4f520ffe7cbc741f4f6080e0855061b0ed54e8be4a84eb9b7", + "h1:U6W8rgrdmR2pZ2cicFoGOSQ4GXuIf/4EK7s0vTJN7is=", + "zh:037fd82cd86227359bc010672cd174235e2d337601d4686f526d0f53c87447cb", + "zh:0ea1db63d6173d01f2fa8eb8989f0809a55135a0d8d424b08ba5dabad73095fa", + "zh:17a4d0a306566f2e45778fbac48744b6fd9c958aaa359e79f144c6358cb93af0", + "zh:298e5408ab17fd2e90d2cd6d406c6d02344fe610de5b7dae943a58b958e76691", + "zh:38ecfd29ee0785fd93164812dcbe0664ebbe5417473f3b2658087ca5a0286ecb", + "zh:59f6a6f31acf66f4ea3667a555a70eba5d406c6e6d93c2c641b81d63261eeace", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:8bb068496b4679bef625e4710d9f3432e301c3a56602271f04e60eadf7f8a94c", - "zh:94742aa5378bab626ce34f79bcef6a373e4f86ea7a8b762e9f71270a899e0d00", - "zh:a485831b5a525cd8f40e8982fa37da40ff70b1ae092c8b755fcde123f0b1238d", - "zh:a647ff16d071eabcabd87ea8183eb90a775a0294ddd735d742075d62fff09193", - "zh:b74710c5954aaa3faf262c18d36a8c2407862d9f842c63e7fa92fa4de3d29df6", - "zh:fa73d83edc92af2e551857594c2232ba6a9e3603ad34b0a5940865202c08d8d7", + "zh:ad0279dfd09d713db0c18469f585e58d04748ca72d9ada83883492e0dd13bd58", + "zh:c69f66fd21f5e2c8ecf7ca68d9091c40f19ad913aef21e3ce23836e91b8cbb5f", + "zh:d4a56f8c48aa86fc8e0c233d56850f5783f322d6336f3bf1916e293246b6b5d4", + "zh:f2b394ebd4af33f343835517e80fc876f79361f4688220833bc3c77655dd2202", + "zh:f31982f29f12834e5d21e010856eddd19d59cd8f449adf470655bfd19354377e", ] } provider "registry.terraform.io/hashicorp/helm" { - version = "2.12.1" - constraints = ">= 2.0.0, ~> 2.12, <= 2.12.1" + version = "2.16.0" + constraints = ">= 2.0.0, ~> 2.12, <= 2.16.0" hashes = [ - "h1:7wfYOAeSEchHB8idNl+2jf+OkFi9zFSOLWkEZFuTCik=", - "h1:aBfcqM4cbywa7TAxfT1YoFS+Cst9waerlm4XErFmJlk=", - "h1:sgYI7lwGqJqPopY3NGmhb1eQ0YbH8PIXaAZAmnJrAvw=", - "h1:sjzfyNQAjtF9zXHxB67geryjGkHaPDMMVw9iqPP5pkE=", - "h1:xwHVa6ab/XVfDrZ3h35OzLJ6g0Zte4VAvSnyKw3f9AI=", - "zh:1d623fb1662703f2feb7860e3c795d849c77640eecbc5a776784d08807b15004", - "zh:253a5bc62ba2c4314875139e3fbd2feaad5ef6b0fb420302a474ab49e8e51a38", - "zh:282358f4ad4f20d0ccaab670b8645228bfad1c03ac0d0df5889f0aea8aeac01a", - "zh:4fd06af3091a382b3f0d8f0a60880f59640d2b6d9d6a31f9a873c6f1bde1ec50", - "zh:6816976b1830f5629ae279569175e88b497abbbac30ee809948a1f923c67a80d", - "zh:7d82c4150cdbf48cfeec867be94c7b9bd7682474d4df0ebb7e24e148f964844f", - "zh:83f062049eea2513118a4c6054fb06c8600bac96196f25aed2cc21898ec86e93", - "zh:a79eec0cf4c08fca79e44033ec6e470f25ff23c3e2c7f9bc707ed7771c1072c0", - "zh:b2b2d904b2821a6e579910320605bc478bbef063579a23fbfdd6fcb5871b81f8", - "zh:e91177ca06a15487fc570cb81ecef6359aa399459ea2aa7c4f7367ba86f6fcad", - "zh:e976bcb82996fc4968f8382bbcb6673efb1f586bf92074058a232028d97825b1", + "h1:uJs402IoDa/7+AnBQZC1txmO0jY4v9W1TMHAvRaCZkY=", + "zh:0fa970817bab7a8411ff443d51004dc2974c0ef4aad082a514f8b56559db3113", + "zh:333b9ac02fcbf9dcf4825dc1e4fc373ef4571b1dd00b79f5c8ea24e1c79992f0", + "zh:792e1e9c409dd76e3eabf3b0c0a6b5a3c3ef42adfc578f7899def46a81e994ef", + "zh:8eca4a52d43ca97d944a8c5d0f2ee60bcbefcb3ccee51d5620bde9047b8ea9c7", + "zh:90969e6a0f7127b0cb75c8790f63f4d050576ffe9bd722887a11d885430624cd", + "zh:a9d72fb106f16ab4f68c779a2c59124929cbc1cb0dbc47ed5ef380c6205f70bb", + "zh:c28bc1a2c0f8f11626baf905a888b2600663ba8dbb33ce4203efcafa16c77fc5", + "zh:c5d6c72a8c5513ff868209ceda9e6000723b02d21811d05909d26614784d4db6", + "zh:d105d40b1a217120332f65a93b24470d18e355868bfa99f0cdeeff5869cff9fb", + "zh:e6c78637c8c6081b8817f61658de8d0163b92157336ac3236cf183b5834f9487", + "zh:edef68729e4f263df3a6737fc73b14e1ee952b800d72d0c6f2cb524bc1ad7ec8", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } provider "registry.terraform.io/hashicorp/kubernetes" { - version = "2.27.0" - constraints = "~> 2.27, <= 2.27.0" + version = "2.33.0" + constraints = "~> 2.27, <= 2.33.0" hashes = [ - "h1:/3kLyOR2jTaWS1MKso4xAztrocGBMxi8yVadWiqSWOg=", - "h1:GzU0FzYAT/+IgAhnSBcFH3bT+4I5N6oSga6iZgNJAus=", - "h1:TrlG/sofnDv8kAbzKOD5pIPeUiI5VQY61NuWH+cItDw=", - "h1:WuU4rl7szPJr9Nfu5OoQGF84k8yQf+gmS9zU2eZuxcc=", - "h1:w9ENsSqT/3Oj/yt4GcudG202ehSD2Ls5gwqOLoKrBUQ=", - "zh:3bdba30ae67c55dc7e9a317ac0da3b208ea7926fe9c2f0ae6587ee88dcc58d1f", - "zh:3f35138a831c00b188d2ffee27111dd0cf59afad2dd5653ed9e67d59646de12c", - "zh:64066d18f6ae9a316c2bc840ef3e641d7ab94e1ea3a41d12523e77345ad442ef", - "zh:653063d44b44881af3a480f7f8eaa94fa300e0229df2072d30f606bddcc9f025", - "zh:87f306e37efb61d13efa6da53a1e45e97e5996ebc0568b1caf8c3c5e54c05809", - "zh:8c428b9708f9634391e52300218771eab3fe942bb1295d8c0ad50ca4b33db3d9", - "zh:a44e87119a0337ded15479851786a13f412b413d9a463ba550d1210249206b0f", - "zh:aa2c4d110b0de6ef997c0d45f3f23f8a98f5530753095d6eff439a6d91a8ea31", - "zh:eb15ed8781ac6a0dec2f7d03cf090e23cfa05e3225806c6231ff2c574662fd63", - "zh:eb81c563f93bd3303f9620d11cd49f21f3f89ac3475c6d3e821b239feb9c217d", - "zh:f1a344a7f16131123577e4ec994d04a34ea458ec16c1ccac53fe7946bd817b18", + "h1:44s6P+u1FUHyEclCAyko9UL+PB73rGp+REnCML3hyzg=", + "zh:255b35790b706d405e987750190658dcaefb663741b96803a9529ba5d7435329", + "zh:362feba1aa820a8e02869ec71d1a08e87243dbce43671dc0995fa6c5a2fafa1d", + "zh:39332abcf75b5dd9c78c79c7c0c094f7d4ca908d1b76bbd2aae67e8e3516710c", + "zh:3e8e7f758bb09a9b5b613c8866e77541f8f00b521070cc86bc095ce61f010baf", + "zh:427883b889b9c36630c3eec4d5c07bc4ae12cc0d358fc17ea42a8049bf8d5275", + "zh:69bfc4ed067a5e4844db1a1809343652ff239aa0a8da089b1671524c44e8740a", + "zh:6b9f731062b945c5020e0930ed9a1b1b50afd2caf751f0e70a282d165c970979", + "zh:6faf9ec006af7ee7014a9c3251d65b701792abb823f149b0b7e4ac4433848201", + "zh:b706f76d695104a47682ee6ab842870f9c70a680f979fa9e7efe34278c0831bc", + "zh:b9bca48de2c92f57389ed58dd2fac564deaccd79a92cafd08edeed3ba6b91d4d", + "zh:bbd3336dbee5aed9880f98e36fb8340e0c6d8f0399a05787521af599ccb3dac4", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } provider "registry.terraform.io/hashicorp/null" { - version = "3.2.2" - constraints = "~> 3.2, <= 3.2.2" + version = "3.2.3" + constraints = "~> 3.2, <= 3.2.3" hashes = [ - "h1:Gef5VGfobY5uokA5nV/zFvWeMNR2Pmq79DH94QnNZPM=", - "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", - "h1:m467k2tZ9cdFFgHW7LPBK2GLPH43LC6wc3ppxr8yvoE=", - "h1:vWAsYRd7MjYr3adj8BVKRohVfHpWQdvkIwUQ2Jf5FVM=", - "h1:zT1ZbegaAYHwQa+QwIFugArWikRJI9dqohj8xb0GY88=", - "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", - "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", - "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", - "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", - "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", - "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", - "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", + "h1:nKUqWEza6Lcv3xRlzeiRQrHtqvzX1BhIzjaOVXRYQXQ=", + "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", + "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", + "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", + "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", + "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", - "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", - "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", - "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", + "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", + "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", + "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", + "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", + "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", + "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", ] } diff --git a/src/domains/paymentoptions-app/.terraform.lock.hcl b/src/domains/paymentoptions-app/.terraform.lock.hcl deleted file mode 100644 index 97cef8e563..0000000000 --- a/src/domains/paymentoptions-app/.terraform.lock.hcl +++ /dev/null @@ -1,102 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/azuread" { - version = "2.47.0" - constraints = "<= 2.47.0" - hashes = [ - "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", - "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", - "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", - "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", - "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", - "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", - "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", - "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", - "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", - "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", - "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", - "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", - "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", - ] -} - -provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.97.1" - constraints = ">= 3.30.0, ~> 3.30, <= 3.97.1, <= 3.106.0" - hashes = [ - "h1:LtwGbd4HEb5QCXmdxSvTjPSh8/Gp8eAQMYfiAKaubV4=", - "zh:15171efcc3aa3a37748c502c493cb16ecff603b81ada4499a843574976bac524", - "zh:2ca6c13a4a96f67763ecced0015c7b101ee02d54ea54b28a8df4ae06468071b1", - "zh:2e3c77dbfd8f760132ecef2d6117e939cbea26b96aba5e4d926e7f7f0f7afe72", - "zh:4bc346eece1622be93c73801d8256502b11fd7c2e7f7cea12d048bb9fc9fe900", - "zh:4f1042942ed8d0433680a367527289459d43b0894a51eaba83ac414e80d5187f", - "zh:63e674c31482ae3579ea84daf5b1ba066ce40cb23475f54e17b6b131320a1bec", - "zh:8327148766dcb7a174673729a832c8095d7e137d0e6c7e2a9a01da48b8b73fbe", - "zh:851b3ae417059a80c7813e7f0063298a590a42f056004f2c2558ea14061c207e", - "zh:ac081b48907139c121a422ae9b1f40fc72c6aaaeb05cbdbf848102a6a5f426f4", - "zh:dc1d663df2d95e4ba91070ceb20d3560b6ea5c465d39c57a5979319302643e41", - "zh:ed26457367cbbb94237e935d297cb31b5687f9abf697377da0ee46974480db9b", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/helm" { - version = "2.12.1" - constraints = "~> 2.12, <= 2.12.1" - hashes = [ - "h1:7wfYOAeSEchHB8idNl+2jf+OkFi9zFSOLWkEZFuTCik=", - "zh:1d623fb1662703f2feb7860e3c795d849c77640eecbc5a776784d08807b15004", - "zh:253a5bc62ba2c4314875139e3fbd2feaad5ef6b0fb420302a474ab49e8e51a38", - "zh:282358f4ad4f20d0ccaab670b8645228bfad1c03ac0d0df5889f0aea8aeac01a", - "zh:4fd06af3091a382b3f0d8f0a60880f59640d2b6d9d6a31f9a873c6f1bde1ec50", - "zh:6816976b1830f5629ae279569175e88b497abbbac30ee809948a1f923c67a80d", - "zh:7d82c4150cdbf48cfeec867be94c7b9bd7682474d4df0ebb7e24e148f964844f", - "zh:83f062049eea2513118a4c6054fb06c8600bac96196f25aed2cc21898ec86e93", - "zh:a79eec0cf4c08fca79e44033ec6e470f25ff23c3e2c7f9bc707ed7771c1072c0", - "zh:b2b2d904b2821a6e579910320605bc478bbef063579a23fbfdd6fcb5871b81f8", - "zh:e91177ca06a15487fc570cb81ecef6359aa399459ea2aa7c4f7367ba86f6fcad", - "zh:e976bcb82996fc4968f8382bbcb6673efb1f586bf92074058a232028d97825b1", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/kubernetes" { - version = "2.29.0" - constraints = "~> 2.27, <= 2.29.0" - hashes = [ - "h1:Igs0JTtmzn5q7RHqrvrTMCD/DCSLPMinvUnhYZ2oITw=", - "zh:3edd5dc319b95fe94e61b82d10c1ce7fb53a2f21b067ddb742f2d7d0d19dd113", - "zh:4b9096e6d0cfa0efd4c89270e3d25fea49db570e2cfbe49c5d1de085a15f2578", - "zh:5397573838bcb8844248c8d6ac93cca7f39a0b707ac3ce7a7b306c50c261c195", - "zh:5d635370720d356b7bcb5756ca28de3275ca32ca1ef0201414caecd3a14759ac", - "zh:71a52280408f3fb0ff1866a9ab8059b0d9bde5481869658798e0773461f22eff", - "zh:748663ef0248d2d95f5dea2974332432a395165657856878c5dc6f000b37cc25", - "zh:7fbc1e084bbbb51e31afd3df0c77e833ae59e88cf42b9e2c17b0b1a1e3894723", - "zh:ae89b4be473b446270fa24dc1ef51b0cc4c2a528d9838ec15246d28bac165df3", - "zh:b6433970d680a0cc9898f915224508b5ece86ae4418372fa6bebd2a9d344f226", - "zh:bf871955cf49015e6a0433e814a22a109c1537a775b8b5dc7b37ad05c324904a", - "zh:c16fac91b2197b443a191d98cf37424feed550387ab11bd1427bde819722005e", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.1" - constraints = "~> 3.2, <= 3.2.1" - hashes = [ - "h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=", - "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", - "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", - "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", - "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", - "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", - "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", - "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", - "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", - "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", - "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", - ] -} diff --git a/src/domains/paymentoptions-app/02_namespace.tf b/src/domains/paymentoptions-app/02_namespace.tf index a62b8cc66c..fe61498c87 100644 --- a/src/domains/paymentoptions-app/02_namespace.tf +++ b/src/domains/paymentoptions-app/02_namespace.tf @@ -4,17 +4,25 @@ resource "kubernetes_namespace" "namespace" { } } -module "pod_identity" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_pod_identity?ref=v8.18.0" +module "workload_identity" { + source = "./.terraform/modules/__v3__/kubernetes_workload_identity_init" - resource_group_name = local.aks_resource_group_name - location = var.location - tenant_id = data.azurerm_subscription.current.tenant_id - cluster_name = local.aks_name + workload_identity_name_prefix = "${var.domain}-workload-identity" + workload_identity_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name + workload_identity_location = var.location +} + +module "workload_identity" { + source = "./.terraform/modules/__v3__/kubernetes_workload_identity_configuration" - identity_name = "${kubernetes_namespace.namespace.metadata[0].name}-pod-identity" - namespace = kubernetes_namespace.namespace.metadata[0].name - key_vault_id = data.azurerm_key_vault.kv.id + workload_identity_name_prefix = "${var.domain}-poc" + workload_identity_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name + aks_name = data.azurerm_kubernetes_cluster.aks.name + aks_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name + namespace = var.domain - secret_permissions = ["Get"] + key_vault_id = data.azurerm_key_vault.kv.id + key_vault_certificate_permissions = ["Get"] + key_vault_key_permissions = ["Get"] + key_vault_secret_permissions = ["Get"] } diff --git a/src/domains/paymentoptions-app/03_serviceaccounts_azure_devops.tf b/src/domains/paymentoptions-app/03_serviceaccounts_azure_devops.tf index f24964a97d..cfdc7c2574 100644 --- a/src/domains/paymentoptions-app/03_serviceaccounts_azure_devops.tf +++ b/src/domains/paymentoptions-app/03_serviceaccounts_azure_devops.tf @@ -5,7 +5,7 @@ resource "kubernetes_namespace" "namespace_system" { } module "kubernetes_service_account" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_service_account?ref=v8.18.0" + source = "./.terraform/modules/__v3__/kubernetes_service_account" name = "azure-devops" namespace = "${var.domain}-system" } diff --git a/src/domains/paymentoptions-app/04_apim_payment_options.tf b/src/domains/paymentoptions-app/04_apim_payment_options.tf index 813fdf05fe..3e7f22da2e 100644 --- a/src/domains/paymentoptions-app/04_apim_payment_options.tf +++ b/src/domains/paymentoptions-app/04_apim_payment_options.tf @@ -6,7 +6,7 @@ locals { } module "apim_payment_options_product" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_product?ref=v8.18.0" + source = "./.terraform/modules/__v3__/api_management_product" count = var.is_feature_enabled.paymentoptions ? 1 : 0 product_id = "pagopa_payment_options" diff --git a/src/domains/paymentoptions-app/04_apim_payment_options_mock.tf b/src/domains/paymentoptions-app/04_apim_payment_options_mock.tf index 329281da0f..e142259ae6 100644 --- a/src/domains/paymentoptions-app/04_apim_payment_options_mock.tf +++ b/src/domains/paymentoptions-app/04_apim_payment_options_mock.tf @@ -6,7 +6,7 @@ locals { } module "apim_payment_options_mock_product" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_product?ref=v8.18.0" + source = "./.terraform/modules/__v3__/api_management_product" count = var.is_feature_enabled.paymentoptions_mock ? 1 : 0 product_id = "pagopa-payment-options-mock" @@ -36,7 +36,7 @@ resource "azurerm_api_management_api_version_set" "payment_options_mock_api" { module "apim_api_pay_opt_mock_api" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_api?ref=v8.18.0" + source = "./.terraform/modules/__v3__/api_management_api" count = var.is_feature_enabled.paymentoptions_mock ? 1 : 0 name = format("%s-pay-opt-mock-api", local.project) diff --git a/src/domains/paymentoptions-app/05_aks_middleware_tools.tf b/src/domains/paymentoptions-app/05_aks_middleware_tools.tf index 83a33add1e..c391298e4f 100644 --- a/src/domains/paymentoptions-app/05_aks_middleware_tools.tf +++ b/src/domains/paymentoptions-app/05_aks_middleware_tools.tf @@ -1,38 +1,39 @@ module "tls_checker" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//tls_checker?ref=v8.22.0" - - https_endpoint = local.domain_hostname - alert_name = local.domain_hostname - alert_enabled = true - helm_chart_present = true - namespace = kubernetes_namespace.namespace.metadata[0].name - location_string = var.location_string - kv_secret_name_for_application_insights_connection_string = "app-insight-connection-string" - application_insights_resource_group = data.azurerm_resource_group.monitor_italy_rg.name - application_insights_id = data.azurerm_application_insights.application_insights_italy.id - application_insights_action_group_ids = [data.azurerm_monitor_action_group.slack.id, data.azurerm_monitor_action_group.email.id] - keyvault_name = data.azurerm_key_vault.kv.name - keyvault_tenant_id = data.azurerm_client_config.current.tenant_id + source = "./.terraform/modules/__v3__/tls_checker" + + https_endpoint = local.domain_hostname + alert_name = local.domain_hostname + alert_enabled = true + helm_chart_present = true + namespace = kubernetes_namespace.namespace.metadata[0].name + location_string = var.location_string + kv_secret_name_for_application_insights_connection_string = "appinsights-instrumentation-key" + application_insights_resource_group = data.azurerm_resource_group.monitor_italy_rg.name + application_insights_id = data.azurerm_application_insights.application_insights_italy.id + application_insights_action_group_ids = [data.azurerm_monitor_action_group.slack.id, data.azurerm_monitor_action_group.email.id] + keyvault_name = data.azurerm_key_vault.kv.name + keyvault_tenant_id = data.azurerm_client_config.current.tenant_id + + workload_identity_enabled = true + workload_identity_service_account_name = module.workload_identity.workload_identity_service_account_name + workload_identity_client_id = module.workload_identity.workload_identity_client_id + + depends_on = [module.workload_identity] } -resource "helm_release" "cert_mounter" { - name = "cert-mounter-blueprint" - repository = "https://pagopa.github.io/aks-helm-cert-mounter-blueprint" - chart = "cert-mounter-blueprint" - version = "1.0.4" - namespace = var.domain - timeout = 120 - force_update = true - - values = [ - templatefile("${path.root}/helm/cert-mounter.yaml.tpl", { - NAMESPACE = var.domain, - DOMAIN = var.domain, - CERTIFICATE_NAME = replace(local.domain_hostname, ".", "-"), - ENV_SHORT = var.env_short, - KV_NAME = data.azurerm_key_vault.kv.name - }) - ] +module "cert_mounter" { + source = "./.terraform/modules/__v3__/cert_mounter" + + namespace = var.domain + certificate_name = replace(local.domain_hostname, ".", "-"), + kv_name = data.azurerm_key_vault.kv.name + tenant_id = data.azurerm_subscription.current.tenant_id + + workload_identity_enabled = true + workload_identity_service_account_name = module.workload_identity.workload_identity_service_account_name + workload_identity_client_id = module.workload_identity.workload_identity_client_id + + depends_on = [module.workload_identity] } resource "helm_release" "reloader" { diff --git a/src/domains/paymentoptions-app/99_main.tf b/src/domains/paymentoptions-app/99_main.tf index 8bf0b91ba8..70cb67af7e 100644 --- a/src/domains/paymentoptions-app/99_main.tf +++ b/src/domains/paymentoptions-app/99_main.tf @@ -3,23 +3,23 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "<= 3.106.0" + version = "<= 3.116.0" } azuread = { source = "hashicorp/azuread" - version = "<= 2.47.0" + version = "<= 3.0.2" } null = { source = "hashicorp/null" - version = "<= 3.2.1" + version = "<= 3.2.3" } kubernetes = { source = "hashicorp/kubernetes" - version = "<= 2.29.0" + version = "<= 2.33.0" } helm = { source = "hashicorp/helm" - version = "<= 2.12.1" + version = "<= 2.16.0" } } @@ -47,3 +47,8 @@ provider "helm" { config_path = "${var.k8s_kube_config_path_prefix}/config-${local.aks_name}" } } + +module "__v3__" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" +} + diff --git a/src/domains/paymentoptions-common/.terraform.lock.hcl b/src/domains/paymentoptions-common/.terraform.lock.hcl deleted file mode 100644 index c1bc80ffb2..0000000000 --- a/src/domains/paymentoptions-common/.terraform.lock.hcl +++ /dev/null @@ -1,65 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/azuread" { - version = "2.47.0" - constraints = "<= 2.47.0" - hashes = [ - "h1:g8+gBFM4QVOEQFqAEs5pR6iXpbGvgPvcEi1evHwziyw=", - "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", - "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", - "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", - "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", - "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", - "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", - "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", - "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", - "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", - "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", - "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", - "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", - "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", - ] -} - -provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.106.0" - constraints = "~> 3.30, <= 3.106.0" - hashes = [ - "h1:6t9Nz9tYAR9BfHZ8yc56m+GKRl0nriwjQ5DyA0/TnCs=", - "h1:Mxe1/I27IZK3BP6cm84Gt0+7PXd2EDaDUMxuljm/rUA=", - "zh:07980d6fdc40c0adb670c8413a5c667917d6dbb51fcedc467c35d64c2f3a1f47", - "zh:2e6e8491b1f089644b0d23f8da83398f1e10cf5a62b16efcef2b5454fe923038", - "zh:450dbd72821c5619cc3bcdc20fdd0e29515147e44b733f9c79d3a75851810055", - "zh:5e234c0a2f3c9677ea72b2a6e6ca90defb99fab29ae565f5d1f70728ba4ba78f", - "zh:83fd042ece6977429d79affd03d6ce963d2f122604dbf15a1abf203d7a7bbc8a", - "zh:93027e1f66b3bf83398d572d4e6f6e7777330c78c54da3226dadd50fd868ada9", - "zh:ae3d1dd66140c303df97d93c47a60f16735ce17cf156f45475dcee4a7360af5b", - "zh:daf9d2eb89e785458a76b88bf2ef0696c472094c77cc9cff3b3ea4b885c5a482", - "zh:dd46370141651e6549da6d85e25c7a6770c47581bbaaa27eda2886d41d849747", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f77405c0d8f6e0d93d9da83256b3b02c164bad4c791ed9604310ff02ae086ad1", - "zh:ffa769147bda833aef8802e3a391bd175ec749862764d61cbdaa8200d5b8f893", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.2" - constraints = "<= 3.2.2" - hashes = [ - "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", - "h1:vWAsYRd7MjYr3adj8BVKRohVfHpWQdvkIwUQ2Jf5FVM=", - "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", - "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", - "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", - "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", - "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", - "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", - "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", - "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", - "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", - "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", - ] -} diff --git a/src/domains/paymentoptions-common/03_eventhub.tf b/src/domains/paymentoptions-common/03_eventhub.tf index b2693b1ba8..e8d3ee1f9e 100644 --- a/src/domains/paymentoptions-common/03_eventhub.tf +++ b/src/domains/paymentoptions-common/03_eventhub.tf @@ -6,7 +6,7 @@ resource "azurerm_resource_group" "eventhub_ita_rg" { } module "eventhub_namespace" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub?ref=v8.22.0" + source = "./.terraform/modules/__v3__/eventhub" name = "${local.project}-evh" location = var.location resource_group_name = azurerm_resource_group.eventhub_ita_rg.name @@ -52,7 +52,7 @@ module "eventhub_namespace" { # CONFIGURATION # module "eventhub_paymentoptions_configuration" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration?ref=v8.22.0" + source = "./.terraform/modules/__v3__/eventhub_configuration" count = var.is_feature_enabled.eventhub ? 1 : 0 event_hub_namespace_name = module.eventhub_namespace.name diff --git a/src/domains/paymentoptions-common/10_github_identity.tf b/src/domains/paymentoptions-common/10_github_identity.tf index 41e54948b3..e77e55ee1a 100644 --- a/src/domains/paymentoptions-common/10_github_identity.tf +++ b/src/domains/paymentoptions-common/10_github_identity.tf @@ -59,7 +59,7 @@ locals { # create a module for each 20 repos module "identity_cd_01" { - source = "github.com/pagopa/terraform-azurerm-v3//github_federated_identity?ref=v8.22.0" + source = "./.terraform/modules/__v3__/github_federated_identity" # pagopa---github--identity prefix = var.prefix env_short = var.env_short @@ -131,7 +131,7 @@ resource "null_resource" "github_runner_app_permissions_to_namespace_cd_01" { # create a module for each 20 repos module "identity_pr_01" { - source = "github.com/pagopa/terraform-azurerm-v3//github_federated_identity?ref=v8.22.0" + source = "./.terraform/modules/__v3__/github_federated_identity" prefix = var.prefix env_short = var.env_short domain = "${var.domain}-01-pr" @@ -170,7 +170,7 @@ resource "azurerm_key_vault_access_policy" "gha_pr_iac_managed_identities" { # create a module for each 20 repos module "identity_ref_01" { - source = "github.com/pagopa/terraform-azurerm-v3//github_federated_identity?ref=v8.36.1" + source = "./.terraform/modules/__v3__/github_federated_identity" prefix = var.prefix env_short = var.env_short domain = "${var.domain}-01-ref" diff --git a/src/domains/paymentoptions-common/99_main.tf b/src/domains/paymentoptions-common/99_main.tf index cb415d65d1..0e2b27945d 100644 --- a/src/domains/paymentoptions-common/99_main.tf +++ b/src/domains/paymentoptions-common/99_main.tf @@ -3,15 +3,15 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "<= 3.106.0" + version = "<= 3.116.0" } azuread = { source = "hashicorp/azuread" - version = "<= 2.47.0" + version = "<= 3.0.2" } null = { source = "hashicorp/null" - version = "<= 3.2.2" + version = "<= 3.2.3" } } @@ -29,3 +29,7 @@ provider "azurerm" { data "azurerm_subscription" "current" {} data "azurerm_client_config" "current" {} + +module "__v3__" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" +} diff --git a/src/domains/paymentoptions-secrets/.terraform.lock.hcl b/src/domains/paymentoptions-secrets/.terraform.lock.hcl deleted file mode 100644 index a389468af5..0000000000 --- a/src/domains/paymentoptions-secrets/.terraform.lock.hcl +++ /dev/null @@ -1,107 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/azuread" { - version = "2.47.0" - constraints = "<= 2.47.0" - hashes = [ - "h1:g8+gBFM4QVOEQFqAEs5pR6iXpbGvgPvcEi1evHwziyw=", - "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", - "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", - "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", - "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", - "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", - "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", - "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", - "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", - "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", - "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", - "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", - "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", - "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", - ] -} - -provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.106.0" - constraints = "~> 3.30, <= 3.106.0" - hashes = [ - "h1:6t9Nz9tYAR9BfHZ8yc56m+GKRl0nriwjQ5DyA0/TnCs=", - "h1:Mxe1/I27IZK3BP6cm84Gt0+7PXd2EDaDUMxuljm/rUA=", - "zh:07980d6fdc40c0adb670c8413a5c667917d6dbb51fcedc467c35d64c2f3a1f47", - "zh:2e6e8491b1f089644b0d23f8da83398f1e10cf5a62b16efcef2b5454fe923038", - "zh:450dbd72821c5619cc3bcdc20fdd0e29515147e44b733f9c79d3a75851810055", - "zh:5e234c0a2f3c9677ea72b2a6e6ca90defb99fab29ae565f5d1f70728ba4ba78f", - "zh:83fd042ece6977429d79affd03d6ce963d2f122604dbf15a1abf203d7a7bbc8a", - "zh:93027e1f66b3bf83398d572d4e6f6e7777330c78c54da3226dadd50fd868ada9", - "zh:ae3d1dd66140c303df97d93c47a60f16735ce17cf156f45475dcee4a7360af5b", - "zh:daf9d2eb89e785458a76b88bf2ef0696c472094c77cc9cff3b3ea4b885c5a482", - "zh:dd46370141651e6549da6d85e25c7a6770c47581bbaaa27eda2886d41d849747", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f77405c0d8f6e0d93d9da83256b3b02c164bad4c791ed9604310ff02ae086ad1", - "zh:ffa769147bda833aef8802e3a391bd175ec749862764d61cbdaa8200d5b8f893", - ] -} - -provider "registry.terraform.io/hashicorp/external" { - version = "2.2.3" - constraints = "<= 2.2.3" - hashes = [ - "h1:648ZjJR81c2W1OLtYmUQa9/1rGr3vvZSuX9dR1ucGWY=", - "h1:D2RKjqoU26isFINpmeKG9NS0LvkPmrQkNXeYO2TdgyA=", - "zh:184ecd339d764de845db0e5b8a9c87893dcd0c9d822167f73658f89d80ec31c9", - "zh:2661eaca31d17d6bbb18a8f673bbfe3fe1b9b7326e60d0ceb302017003274e3c", - "zh:2c0a180f6d1fc2ba6e03f7dfc5f73b617e45408681f75bca75aa82f3796df0e4", - "zh:4b92ae44c6baef4c4952c47be00541055cb5280dd3bc8031dba5a1b2ee982387", - "zh:5641694d5daf3893d7ea90be03b6fa575211a08814ffe70998d5adb8b59cdc0a", - "zh:5bd55a2be8a1c20d732ac9c604b839e1cadc8c49006315dffa4d709b6874df32", - "zh:6e0ef5d11e1597202424b7d69b9da7b881494c9b13a3d4026fc47012dc651c79", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9e19f89fa25004d3b926a8d15ea630b4bde62f1fa4ed5e11a3d27aabddb77353", - "zh:b763efdd69fd097616b4a4c89cf333b4cee9699ac6432d73d2756f8335d1213f", - "zh:e3b561efdee510b2b445f76a52a902c52bee8e13095e7f4bed7c80f10f8d294a", - "zh:fe660bb8781ee043a093b9a20e53069974475dcaa5791a1f45fd03c61a26478a", - ] -} - -provider "registry.terraform.io/hashicorp/kubernetes" { - version = "2.16.1" - constraints = "<= 2.16.1" - hashes = [ - "h1:PO4Ye/+lu5hCaUEOtwNOldQYoA0dqL1bcBICIpdlcd8=", - "h1:kO/d+ZMZYM2tNMMFHZqBmVR0MeemoGnI2G2NSN92CrU=", - "zh:06224975f5910d41e73b35a4d5079861da2c24f9353e3ebb015fbb3b3b996b1c", - "zh:2bc400a8d9fe7755cca27c2551564a9e2609cfadc77f526ef855114ee02d446f", - "zh:3a479014187af1d0aec3a1d3d9c09551b801956fe6dd29af1186dec86712731b", - "zh:73fb0a69f1abdb02858b6589f7fab6d989a0f422f7ad95ed662aaa84872d3473", - "zh:a33852cd382cbc8e06d3f6c018b468ad809d24d912d64722e037aed1f9bf39db", - "zh:b533ff2214dca90296b1d22eace7eaa7e3efe5a7ae9da66a112094abc932db4f", - "zh:ddf74d8bb1aeb01dc2c36ef40e2b283d32b2a96db73f6daaf179fa2f10949c80", - "zh:e720f3a15d34e795fa9ff90bc755e838ebb4aef894aa2a423fb16dfa6d6b0667", - "zh:e789ae70a658800cb0a19ef7e4e9b26b5a38a92b43d1f41d64fc8bb46539cefb", - "zh:e8aed7dc0bd8f843d607dee5f72640dbef6835a8b1c6ea12cea5b4ec53e463f7", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:fb3ac4f43c8b0dfc0b0103dd0f062ea72b3a34518d4c8808e3a44c9a3dd5f024", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.1" - constraints = "~> 3.2, <= 3.2.1" - hashes = [ - "h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=", - "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", - "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", - "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", - "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", - "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", - "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", - "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", - "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", - "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", - "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", - "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", - ] -} diff --git a/src/domains/paymentoptions-secrets/01_keyvault.tf b/src/domains/paymentoptions-secrets/01_keyvault.tf index 28af4e56e2..c91ffe6e9c 100644 --- a/src/domains/paymentoptions-secrets/01_keyvault.tf +++ b/src/domains/paymentoptions-secrets/01_keyvault.tf @@ -6,7 +6,7 @@ resource "azurerm_resource_group" "sec_rg" { } module "key_vault" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//key_vault?ref=v8.22.0" + source = "./.terraform/modules/__v3__/key_vault" name = "${local.product}-${var.location_short}-${var.domain}-kv" location = azurerm_resource_group.sec_rg.location @@ -92,7 +92,7 @@ resource "azurerm_key_vault_access_policy" "azdevops_iac_policy" { # create json letsencrypt inside kv # requierd: Docker module "letsencrypt_paymentoptions" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git///letsencrypt_credential?ref=v8.44.0" + source = "./.terraform/modules/__v3__/letsencrypt_credential" prefix = var.prefix env = var.env_short diff --git a/src/domains/paymentoptions-secrets/99_main.tf b/src/domains/paymentoptions-secrets/99_main.tf index 93ec8f61ca..ba2156d6dd 100644 --- a/src/domains/paymentoptions-secrets/99_main.tf +++ b/src/domains/paymentoptions-secrets/99_main.tf @@ -2,23 +2,23 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "<= 3.106.0" + version = "<= 3.116.0" } azuread = { source = "hashicorp/azuread" - version = "<= 2.47.0" + version = "<= 3.0.2" } null = { source = "hashicorp/null" - version = "<= 3.2.1" + version = "<= 3.2.3" } external = { source = "hashicorp/external" - version = "<= 2.2.3" + version = "<= 2.3.4" } kubernetes = { source = "hashicorp/kubernetes" - version = "<= 2.16.1" + version = "<= 2.33.0" } } @@ -41,3 +41,7 @@ provider "kubernetes" { data "azurerm_subscription" "current" {} data "azurerm_client_config" "current" {} + +module "__v3__" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" +} From c9ef7119c5f4779c11f325e1925c7ecff4d5b86c Mon Sep 17 00:00:00 2001 From: acialini Date: Mon, 14 Oct 2024 11:49:37 +0200 Subject: [PATCH 02/55] [PPANTT-137] feat: updated modules and workload identity for aks-leonardo and payopts domain --- src/aks-leonardo/03_aks_0.tf | 5 ++++- src/aks-leonardo/03_aks_storage.tf | 2 +- src/aks-leonardo/03_monitoring.tf | 4 ++-- src/aks-leonardo/99_main.tf | 16 ++++++++++------ 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/aks-leonardo/03_aks_0.tf b/src/aks-leonardo/03_aks_0.tf index 62397c2e95..eb01a03159 100644 --- a/src/aks-leonardo/03_aks_0.tf +++ b/src/aks-leonardo/03_aks_0.tf @@ -6,7 +6,7 @@ resource "azurerm_resource_group" "rg_aks" { } module "aks_leonardo" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_cluster?ref=v8.20.1" + source = "./.terraform/modules/__v3__/kubernetes_cluster" name = local.aks_cluster_name location = var.location @@ -31,6 +31,9 @@ module "aks_leonardo" { system_node_pool_node_labels = var.aks_system_node_pool.node_labels system_node_pool_tags = var.aks_system_node_pool.node_tags + workload_identity_enabled = var.env_short == "d" ? true : false + oidc_issuer_enabled = var.env_short == "d" ? true : false + # # ☁️ Network # diff --git a/src/aks-leonardo/03_aks_storage.tf b/src/aks-leonardo/03_aks_storage.tf index 93ca67ec3a..9d0e78d054 100644 --- a/src/aks-leonardo/03_aks_storage.tf +++ b/src/aks-leonardo/03_aks_storage.tf @@ -1,3 +1,3 @@ module "aks_storage_class" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_storage_class?ref=v8.17.1" + source = "./.terraform/modules/__v3__/kubernetes_storage_class" } diff --git a/src/aks-leonardo/03_monitoring.tf b/src/aks-leonardo/03_monitoring.tf index 537b2ab730..3aab21fd9f 100644 --- a/src/aks-leonardo/03_monitoring.tf +++ b/src/aks-leonardo/03_monitoring.tf @@ -5,7 +5,7 @@ resource "kubernetes_namespace" "monitoring" { } module "aks_prometheus_install" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_prometheus_install?ref=v8.17.1" + source = "./.terraform/modules/__v3__/kubernetes_prometheus_install" prometheus_namespace = kubernetes_namespace.monitoring.metadata[0].name storage_class_name = "default-zrs" @@ -13,7 +13,7 @@ module "aks_prometheus_install" { module "elastic_agent" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//elastic_agent?ref=introducing-agent-module" + source = "./.terraform/modules/__v3__/elastic_agent" es_host = var.env == "p" ? "https://weu${var.env}.kibana.internal.platform.pagopa.it:443/elastic" : "https://weu${var.env}.kibana.internal.${var.env}.platform.pagopa.it:443/elastic" diff --git a/src/aks-leonardo/99_main.tf b/src/aks-leonardo/99_main.tf index be793930c3..dc6789fcab 100644 --- a/src/aks-leonardo/99_main.tf +++ b/src/aks-leonardo/99_main.tf @@ -3,27 +3,27 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "<= 3.105.0" + version = "<= 3.116.0" } azuread = { source = "hashicorp/azuread" - version = "<= 2.47.0" + version = "<= 3.0.2" } external = { source = "hashicorp/external" - version = "<= 2.3.3" + version = "<= 2.3.4" } kubernetes = { source = "hashicorp/kubernetes" - version = "<= 2.27.0" + version = "<= 2.33.0" } helm = { source = "hashicorp/helm" - version = "<= 2.12.1" + version = "<= 2.16.0" } null = { source = "hashicorp/null" - version = "<= 3.2.2" + version = "<= 3.2.3" } } @@ -51,3 +51,7 @@ provider "helm" { config_path = "${var.k8s_kube_config_path_prefix}/config-${local.aks_cluster_name}" } } + +module "__v3__" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" +} From 1c9537f7e38ce120165a30660d8ee6c45d0c0ae2 Mon Sep 17 00:00:00 2001 From: acialini Date: Mon, 21 Oct 2024 12:43:21 +0200 Subject: [PATCH 03/55] [PPANTT-168] feat: Introducing infra domain for gpd ingestion --- .../gpdingestion-app/.terraform.lock.hcl | 102 ++++++ src/domains/gpdingestion-app/00_data.tf | 9 + src/domains/gpdingestion-app/00_keyvault.tf | 10 + src/domains/gpdingestion-app/00_monitor.tf | 35 ++ src/domains/gpdingestion-app/00_network.tf | 15 + src/domains/gpdingestion-app/01_network.tf | 9 + src/domains/gpdingestion-app/02_namespace.tf | 20 ++ .../03_serviceaccounts_azure_devops.tf | 67 ++++ .../05_aks_middleware_tools.tf | 49 +++ src/domains/gpdingestion-app/06_keyvault.tf | 38 ++ src/domains/gpdingestion-app/90_pdb.tf | 15 + src/domains/gpdingestion-app/99_locals.tf | 43 +++ src/domains/gpdingestion-app/99_main.tf | 49 +++ src/domains/gpdingestion-app/99_variables.tf | 158 +++++++++ src/domains/gpdingestion-app/README.md | 106 ++++++ .../gpdingestion-app/env/itn-dev/backend.ini | 1 + .../env/itn-dev/backend.tfvars | 4 + .../env/itn-dev/terraform.tfvars | 38 ++ .../gpdingestion-app/env/itn-prod/backend.ini | 1 + .../env/itn-prod/backend.tfvars | 4 + .../env/itn-prod/terraform.tfvars | 47 +++ .../gpdingestion-app/env/itn-uat/backend.ini | 1 + .../env/itn-uat/backend.tfvars | 4 + .../env/itn-uat/terraform.tfvars | 38 ++ .../helm/cert-mounter.yaml.tpl | 13 + src/domains/gpdingestion-app/terraform.sh | 324 ++++++++++++++++++ .../gpdingestion-common/.terraform.lock.hcl | 65 ++++ src/domains/gpdingestion-common/00_data.tf | 4 + src/domains/gpdingestion-common/00_monitor.tf | 45 +++ src/domains/gpdingestion-common/00_network.tf | 37 ++ src/domains/gpdingestion-common/01_network.tf | 14 + .../gpdingestion-common/03_eventhub.tf | 63 ++++ .../gpdingestion-common/10_github_identity.tf | 207 +++++++++++ src/domains/gpdingestion-common/99_locals.tf | 32 ++ src/domains/gpdingestion-common/99_main.tf | 31 ++ .../gpdingestion-common/99_variables.tf | 219 ++++++++++++ src/domains/gpdingestion-common/README.md | 90 +++++ .../env/itn-dev/backend.ini | 1 + .../env/itn-dev/backend.tfvars | 4 + .../env/itn-dev/terraform.tfvars | 59 ++++ .../env/itn-prod/backend.ini | 1 + .../env/itn-prod/backend.tfvars | 4 + .../env/itn-prod/terraform.tfvars | 58 ++++ .../env/itn-uat/backend.ini | 1 + .../env/itn-uat/backend.tfvars | 4 + .../env/itn-uat/terraform.tfvars | 58 ++++ src/domains/gpdingestion-common/terraform.sh | 324 ++++++++++++++++++ .../gpdingestion-secrets/.terraform.lock.hcl | 107 ++++++ .../gpdingestion-secrets/00_azuread.tf | 16 + .../gpdingestion-secrets/01_keyvault.tf | 101 ++++++ src/domains/gpdingestion-secrets/02_azdo.tf | 23 ++ .../gpdingestion-secrets/02_init_sops.tf | 21 ++ .../gpdingestion-secrets/03_sops_secrets.tf | 54 +++ src/domains/gpdingestion-secrets/99_locals.tf | 11 + src/domains/gpdingestion-secrets/99_main.tf | 43 +++ .../gpdingestion-secrets/99_variables.tf | 101 ++++++ src/domains/gpdingestion-secrets/README.md | 65 ++++ .../env/itn-dev/backend.ini | 1 + .../env/itn-dev/backend.tfvars | 4 + .../env/itn-dev/terraform.tfvars | 30 ++ .../env/itn-prod/backend.ini | 1 + .../env/itn-prod/backend.tfvars | 4 + .../env/itn-prod/terraform.tfvars | 30 ++ .../env/itn-uat/backend.ini | 1 + .../env/itn-uat/backend.tfvars | 4 + .../env/itn-uat/terraform.tfvars | 27 ++ .../secret/itn-dev/configs.json | 1 + .../secret/itn-dev/secret.ini | 3 + .../secret/itn-prod/configs.json | 2 + .../secret/itn-prod/secret.ini | 3 + .../secret/itn-uat/configs.json | 2 + .../secret/itn-uat/secret.ini | 3 + src/domains/gpdingestion-secrets/sops.sh | 137 ++++++++ src/domains/gpdingestion-secrets/terraform.sh | 324 ++++++++++++++++++ src/domains/gpdingestion-secrets/terrasops.sh | 29 ++ 75 files changed, 3669 insertions(+) create mode 100644 src/domains/gpdingestion-app/.terraform.lock.hcl create mode 100644 src/domains/gpdingestion-app/00_data.tf create mode 100644 src/domains/gpdingestion-app/00_keyvault.tf create mode 100644 src/domains/gpdingestion-app/00_monitor.tf create mode 100644 src/domains/gpdingestion-app/00_network.tf create mode 100644 src/domains/gpdingestion-app/01_network.tf create mode 100644 src/domains/gpdingestion-app/02_namespace.tf create mode 100644 src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf create mode 100644 src/domains/gpdingestion-app/05_aks_middleware_tools.tf create mode 100644 src/domains/gpdingestion-app/06_keyvault.tf create mode 100644 src/domains/gpdingestion-app/90_pdb.tf create mode 100644 src/domains/gpdingestion-app/99_locals.tf create mode 100644 src/domains/gpdingestion-app/99_main.tf create mode 100644 src/domains/gpdingestion-app/99_variables.tf create mode 100644 src/domains/gpdingestion-app/README.md create mode 100644 src/domains/gpdingestion-app/env/itn-dev/backend.ini create mode 100644 src/domains/gpdingestion-app/env/itn-dev/backend.tfvars create mode 100644 src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars create mode 100644 src/domains/gpdingestion-app/env/itn-prod/backend.ini create mode 100644 src/domains/gpdingestion-app/env/itn-prod/backend.tfvars create mode 100644 src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars create mode 100644 src/domains/gpdingestion-app/env/itn-uat/backend.ini create mode 100644 src/domains/gpdingestion-app/env/itn-uat/backend.tfvars create mode 100644 src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars create mode 100644 src/domains/gpdingestion-app/helm/cert-mounter.yaml.tpl create mode 100755 src/domains/gpdingestion-app/terraform.sh create mode 100644 src/domains/gpdingestion-common/.terraform.lock.hcl create mode 100644 src/domains/gpdingestion-common/00_data.tf create mode 100644 src/domains/gpdingestion-common/00_monitor.tf create mode 100644 src/domains/gpdingestion-common/00_network.tf create mode 100644 src/domains/gpdingestion-common/01_network.tf create mode 100644 src/domains/gpdingestion-common/03_eventhub.tf create mode 100644 src/domains/gpdingestion-common/10_github_identity.tf create mode 100644 src/domains/gpdingestion-common/99_locals.tf create mode 100644 src/domains/gpdingestion-common/99_main.tf create mode 100644 src/domains/gpdingestion-common/99_variables.tf create mode 100644 src/domains/gpdingestion-common/README.md create mode 100644 src/domains/gpdingestion-common/env/itn-dev/backend.ini create mode 100644 src/domains/gpdingestion-common/env/itn-dev/backend.tfvars create mode 100644 src/domains/gpdingestion-common/env/itn-dev/terraform.tfvars create mode 100644 src/domains/gpdingestion-common/env/itn-prod/backend.ini create mode 100644 src/domains/gpdingestion-common/env/itn-prod/backend.tfvars create mode 100644 src/domains/gpdingestion-common/env/itn-prod/terraform.tfvars create mode 100644 src/domains/gpdingestion-common/env/itn-uat/backend.ini create mode 100644 src/domains/gpdingestion-common/env/itn-uat/backend.tfvars create mode 100644 src/domains/gpdingestion-common/env/itn-uat/terraform.tfvars create mode 100755 src/domains/gpdingestion-common/terraform.sh create mode 100644 src/domains/gpdingestion-secrets/.terraform.lock.hcl create mode 100644 src/domains/gpdingestion-secrets/00_azuread.tf create mode 100644 src/domains/gpdingestion-secrets/01_keyvault.tf create mode 100644 src/domains/gpdingestion-secrets/02_azdo.tf create mode 100644 src/domains/gpdingestion-secrets/02_init_sops.tf create mode 100644 src/domains/gpdingestion-secrets/03_sops_secrets.tf create mode 100644 src/domains/gpdingestion-secrets/99_locals.tf create mode 100644 src/domains/gpdingestion-secrets/99_main.tf create mode 100644 src/domains/gpdingestion-secrets/99_variables.tf create mode 100644 src/domains/gpdingestion-secrets/README.md create mode 100644 src/domains/gpdingestion-secrets/env/itn-dev/backend.ini create mode 100644 src/domains/gpdingestion-secrets/env/itn-dev/backend.tfvars create mode 100644 src/domains/gpdingestion-secrets/env/itn-dev/terraform.tfvars create mode 100644 src/domains/gpdingestion-secrets/env/itn-prod/backend.ini create mode 100644 src/domains/gpdingestion-secrets/env/itn-prod/backend.tfvars create mode 100644 src/domains/gpdingestion-secrets/env/itn-prod/terraform.tfvars create mode 100644 src/domains/gpdingestion-secrets/env/itn-uat/backend.ini create mode 100644 src/domains/gpdingestion-secrets/env/itn-uat/backend.tfvars create mode 100644 src/domains/gpdingestion-secrets/env/itn-uat/terraform.tfvars create mode 100644 src/domains/gpdingestion-secrets/secret/itn-dev/configs.json create mode 100644 src/domains/gpdingestion-secrets/secret/itn-dev/secret.ini create mode 100644 src/domains/gpdingestion-secrets/secret/itn-prod/configs.json create mode 100644 src/domains/gpdingestion-secrets/secret/itn-prod/secret.ini create mode 100644 src/domains/gpdingestion-secrets/secret/itn-uat/configs.json create mode 100644 src/domains/gpdingestion-secrets/secret/itn-uat/secret.ini create mode 100755 src/domains/gpdingestion-secrets/sops.sh create mode 100755 src/domains/gpdingestion-secrets/terraform.sh create mode 100644 src/domains/gpdingestion-secrets/terrasops.sh diff --git a/src/domains/gpdingestion-app/.terraform.lock.hcl b/src/domains/gpdingestion-app/.terraform.lock.hcl new file mode 100644 index 0000000000..97cef8e563 --- /dev/null +++ b/src/domains/gpdingestion-app/.terraform.lock.hcl @@ -0,0 +1,102 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "2.47.0" + constraints = "<= 2.47.0" + hashes = [ + "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", + "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", + "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", + "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", + "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", + "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", + "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", + "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", + "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", + "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", + "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.97.1" + constraints = ">= 3.30.0, ~> 3.30, <= 3.97.1, <= 3.106.0" + hashes = [ + "h1:LtwGbd4HEb5QCXmdxSvTjPSh8/Gp8eAQMYfiAKaubV4=", + "zh:15171efcc3aa3a37748c502c493cb16ecff603b81ada4499a843574976bac524", + "zh:2ca6c13a4a96f67763ecced0015c7b101ee02d54ea54b28a8df4ae06468071b1", + "zh:2e3c77dbfd8f760132ecef2d6117e939cbea26b96aba5e4d926e7f7f0f7afe72", + "zh:4bc346eece1622be93c73801d8256502b11fd7c2e7f7cea12d048bb9fc9fe900", + "zh:4f1042942ed8d0433680a367527289459d43b0894a51eaba83ac414e80d5187f", + "zh:63e674c31482ae3579ea84daf5b1ba066ce40cb23475f54e17b6b131320a1bec", + "zh:8327148766dcb7a174673729a832c8095d7e137d0e6c7e2a9a01da48b8b73fbe", + "zh:851b3ae417059a80c7813e7f0063298a590a42f056004f2c2558ea14061c207e", + "zh:ac081b48907139c121a422ae9b1f40fc72c6aaaeb05cbdbf848102a6a5f426f4", + "zh:dc1d663df2d95e4ba91070ceb20d3560b6ea5c465d39c57a5979319302643e41", + "zh:ed26457367cbbb94237e935d297cb31b5687f9abf697377da0ee46974480db9b", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.12.1" + constraints = "~> 2.12, <= 2.12.1" + hashes = [ + "h1:7wfYOAeSEchHB8idNl+2jf+OkFi9zFSOLWkEZFuTCik=", + "zh:1d623fb1662703f2feb7860e3c795d849c77640eecbc5a776784d08807b15004", + "zh:253a5bc62ba2c4314875139e3fbd2feaad5ef6b0fb420302a474ab49e8e51a38", + "zh:282358f4ad4f20d0ccaab670b8645228bfad1c03ac0d0df5889f0aea8aeac01a", + "zh:4fd06af3091a382b3f0d8f0a60880f59640d2b6d9d6a31f9a873c6f1bde1ec50", + "zh:6816976b1830f5629ae279569175e88b497abbbac30ee809948a1f923c67a80d", + "zh:7d82c4150cdbf48cfeec867be94c7b9bd7682474d4df0ebb7e24e148f964844f", + "zh:83f062049eea2513118a4c6054fb06c8600bac96196f25aed2cc21898ec86e93", + "zh:a79eec0cf4c08fca79e44033ec6e470f25ff23c3e2c7f9bc707ed7771c1072c0", + "zh:b2b2d904b2821a6e579910320605bc478bbef063579a23fbfdd6fcb5871b81f8", + "zh:e91177ca06a15487fc570cb81ecef6359aa399459ea2aa7c4f7367ba86f6fcad", + "zh:e976bcb82996fc4968f8382bbcb6673efb1f586bf92074058a232028d97825b1", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.29.0" + constraints = "~> 2.27, <= 2.29.0" + hashes = [ + "h1:Igs0JTtmzn5q7RHqrvrTMCD/DCSLPMinvUnhYZ2oITw=", + "zh:3edd5dc319b95fe94e61b82d10c1ce7fb53a2f21b067ddb742f2d7d0d19dd113", + "zh:4b9096e6d0cfa0efd4c89270e3d25fea49db570e2cfbe49c5d1de085a15f2578", + "zh:5397573838bcb8844248c8d6ac93cca7f39a0b707ac3ce7a7b306c50c261c195", + "zh:5d635370720d356b7bcb5756ca28de3275ca32ca1ef0201414caecd3a14759ac", + "zh:71a52280408f3fb0ff1866a9ab8059b0d9bde5481869658798e0773461f22eff", + "zh:748663ef0248d2d95f5dea2974332432a395165657856878c5dc6f000b37cc25", + "zh:7fbc1e084bbbb51e31afd3df0c77e833ae59e88cf42b9e2c17b0b1a1e3894723", + "zh:ae89b4be473b446270fa24dc1ef51b0cc4c2a528d9838ec15246d28bac165df3", + "zh:b6433970d680a0cc9898f915224508b5ece86ae4418372fa6bebd2a9d344f226", + "zh:bf871955cf49015e6a0433e814a22a109c1537a775b8b5dc7b37ad05c324904a", + "zh:c16fac91b2197b443a191d98cf37424feed550387ab11bd1427bde819722005e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.1" + constraints = "~> 3.2, <= 3.2.1" + hashes = [ + "h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=", + "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", + "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", + "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", + "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", + "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", + "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", + "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", + "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", + "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", + "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", + ] +} diff --git a/src/domains/gpdingestion-app/00_data.tf b/src/domains/gpdingestion-app/00_data.tf new file mode 100644 index 0000000000..f05bb6d119 --- /dev/null +++ b/src/domains/gpdingestion-app/00_data.tf @@ -0,0 +1,9 @@ +### EVH +resource "azurerm_eventhub_authorization_rule" "cdc_connection_string" { + name = "cdc-connection-string" + namespace_name = "${local.project}-evh" + resource_group_name = "${local.project}-evh-rg" + listen = true + send = true + manage = true +} diff --git a/src/domains/gpdingestion-app/00_keyvault.tf b/src/domains/gpdingestion-app/00_keyvault.tf new file mode 100644 index 0000000000..c94a899cca --- /dev/null +++ b/src/domains/gpdingestion-app/00_keyvault.tf @@ -0,0 +1,10 @@ +data "azurerm_key_vault" "kv" { + name = "${local.project}-kv" + resource_group_name = "${local.project}-sec-rg" +} + + +data "azurerm_kubernetes_cluster" "aks" { + name = local.aks_name + resource_group_name = local.aks_resource_group_name +} diff --git a/src/domains/gpdingestion-app/00_monitor.tf b/src/domains/gpdingestion-app/00_monitor.tf new file mode 100644 index 0000000000..311dc4ff7d --- /dev/null +++ b/src/domains/gpdingestion-app/00_monitor.tf @@ -0,0 +1,35 @@ +# +# 🇮🇹 Monitor Italy +# +data "azurerm_resource_group" "monitor_italy_rg" { + name = var.monitor_italy_resource_group_name +} + +data "azurerm_log_analytics_workspace" "log_analytics_italy" { + name = var.log_analytics_italy_workspace_name + resource_group_name = var.log_analytics_italy_workspace_resource_group_name +} + +data "azurerm_application_insights" "application_insights_italy" { + name = local.monitor_appinsights_italy_name + resource_group_name = data.azurerm_resource_group.monitor_italy_rg.name +} + +# +# Actions Group +# +data "azurerm_monitor_action_group" "slack" { + name = local.monitor_action_group_slack_name + resource_group_name = var.monitor_italy_resource_group_name +} + +data "azurerm_monitor_action_group" "email" { + resource_group_name = var.monitor_italy_resource_group_name + name = local.monitor_action_group_email_name +} + +data "azurerm_monitor_action_group" "opsgenie" { + count = var.env_short == "p" ? 1 : 0 + resource_group_name = var.monitor_resource_group_name + name = local.monitor_action_group_opsgenie_name +} diff --git a/src/domains/gpdingestion-app/00_network.tf b/src/domains/gpdingestion-app/00_network.tf new file mode 100644 index 0000000000..355c8e2333 --- /dev/null +++ b/src/domains/gpdingestion-app/00_network.tf @@ -0,0 +1,15 @@ +data "azurerm_virtual_network" "vnet" { + name = local.vnet_name + resource_group_name = local.vnet_resource_group_name +} + +data "azurerm_private_dns_zone" "internal" { + name = local.internal_dns_zone_name + resource_group_name = local.internal_dns_zone_resource_group_name +} + +data "azurerm_subnet" "apim_vnet" { + name = local.pagopa_apim_snet + resource_group_name = local.pagopa_vnet_rg + virtual_network_name = local.pagopa_vnet_integration +} diff --git a/src/domains/gpdingestion-app/01_network.tf b/src/domains/gpdingestion-app/01_network.tf new file mode 100644 index 0000000000..73614770ca --- /dev/null +++ b/src/domains/gpdingestion-app/01_network.tf @@ -0,0 +1,9 @@ +#-------------------------------------------------- + +resource "azurerm_private_dns_a_record" "ingress" { + name = local.ingress_hostname + zone_name = data.azurerm_private_dns_zone.internal.name + resource_group_name = local.internal_dns_zone_resource_group_name + ttl = 3600 + records = [var.ingress_load_balancer_ip] +} diff --git a/src/domains/gpdingestion-app/02_namespace.tf b/src/domains/gpdingestion-app/02_namespace.tf new file mode 100644 index 0000000000..a62b8cc66c --- /dev/null +++ b/src/domains/gpdingestion-app/02_namespace.tf @@ -0,0 +1,20 @@ +resource "kubernetes_namespace" "namespace" { + metadata { + name = var.domain + } +} + +module "pod_identity" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_pod_identity?ref=v8.18.0" + + resource_group_name = local.aks_resource_group_name + location = var.location + tenant_id = data.azurerm_subscription.current.tenant_id + cluster_name = local.aks_name + + identity_name = "${kubernetes_namespace.namespace.metadata[0].name}-pod-identity" + namespace = kubernetes_namespace.namespace.metadata[0].name + key_vault_id = data.azurerm_key_vault.kv.id + + secret_permissions = ["Get"] +} diff --git a/src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf b/src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf new file mode 100644 index 0000000000..f24964a97d --- /dev/null +++ b/src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf @@ -0,0 +1,67 @@ +resource "kubernetes_namespace" "namespace_system" { + metadata { + name = "${var.domain}-system" + } +} + +module "kubernetes_service_account" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_service_account?ref=v8.18.0" + name = "azure-devops" + namespace = "${var.domain}-system" +} + +#tfsec:ignore:AZU023 +resource "azurerm_key_vault_secret" "azure_devops_sa_token" { + depends_on = [module.kubernetes_service_account] + name = "${local.aks_name}-azure-devops-sa-token" + value = module.kubernetes_service_account.sa_token # base64 value + content_type = "text/plain" + + key_vault_id = data.azurerm_key_vault.kv.id +} + +#tfsec:ignore:AZU023 +resource "azurerm_key_vault_secret" "azure_devops_sa_cacrt" { + depends_on = [module.kubernetes_service_account] + name = "${local.aks_name}-azure-devops-sa-cacrt" + value = module.kubernetes_service_account.sa_ca_cert # base64 value + content_type = "text/plain" + + key_vault_id = data.azurerm_key_vault.kv.id +} + +#-------------------------------------------------------------------------------------------------- + +resource "kubernetes_role_binding" "deployer_binding" { + metadata { + name = "deployer-binding" + namespace = kubernetes_namespace.namespace.metadata[0].name + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "cluster-deployer" + } + subject { + kind = "ServiceAccount" + name = "azure-devops" + namespace = kubernetes_namespace.namespace_system.metadata[0].name + } +} + +resource "kubernetes_role_binding" "system_deployer_binding" { + metadata { + name = "system-deployer-binding" + namespace = kubernetes_namespace.namespace_system.metadata[0].name + } + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = "system-cluster-deployer" + } + subject { + kind = "ServiceAccount" + name = "azure-devops" + namespace = kubernetes_namespace.namespace_system.metadata[0].name + } +} diff --git a/src/domains/gpdingestion-app/05_aks_middleware_tools.tf b/src/domains/gpdingestion-app/05_aks_middleware_tools.tf new file mode 100644 index 0000000000..83a33add1e --- /dev/null +++ b/src/domains/gpdingestion-app/05_aks_middleware_tools.tf @@ -0,0 +1,49 @@ +module "tls_checker" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//tls_checker?ref=v8.22.0" + + https_endpoint = local.domain_hostname + alert_name = local.domain_hostname + alert_enabled = true + helm_chart_present = true + namespace = kubernetes_namespace.namespace.metadata[0].name + location_string = var.location_string + kv_secret_name_for_application_insights_connection_string = "app-insight-connection-string" + application_insights_resource_group = data.azurerm_resource_group.monitor_italy_rg.name + application_insights_id = data.azurerm_application_insights.application_insights_italy.id + application_insights_action_group_ids = [data.azurerm_monitor_action_group.slack.id, data.azurerm_monitor_action_group.email.id] + keyvault_name = data.azurerm_key_vault.kv.name + keyvault_tenant_id = data.azurerm_client_config.current.tenant_id +} + +resource "helm_release" "cert_mounter" { + name = "cert-mounter-blueprint" + repository = "https://pagopa.github.io/aks-helm-cert-mounter-blueprint" + chart = "cert-mounter-blueprint" + version = "1.0.4" + namespace = var.domain + timeout = 120 + force_update = true + + values = [ + templatefile("${path.root}/helm/cert-mounter.yaml.tpl", { + NAMESPACE = var.domain, + DOMAIN = var.domain, + CERTIFICATE_NAME = replace(local.domain_hostname, ".", "-"), + ENV_SHORT = var.env_short, + KV_NAME = data.azurerm_key_vault.kv.name + }) + ] +} + +resource "helm_release" "reloader" { + name = "reloader" + repository = "https://stakater.github.io/stakater-charts" + chart = "reloader" + version = "v1.0.69" + namespace = kubernetes_namespace.namespace.metadata[0].name + + set { + name = "reloader.watchGlobally" + value = "false" + } +} diff --git a/src/domains/gpdingestion-app/06_keyvault.tf b/src/domains/gpdingestion-app/06_keyvault.tf new file mode 100644 index 0000000000..152ae794da --- /dev/null +++ b/src/domains/gpdingestion-app/06_keyvault.tf @@ -0,0 +1,38 @@ +locals { + aks_api_url = var.env_short == "d" ? data.azurerm_kubernetes_cluster.aks.fqdn : data.azurerm_kubernetes_cluster.aks.private_fqdn +} + +#tfsec:ignore:AZU023 +resource "azurerm_key_vault_secret" "aks_apiserver_url" { + name = "${local.aks_name}-apiserver-url" + value = "https://${local.aks_api_url}:443" + content_type = "text/plain" + + key_vault_id = data.azurerm_key_vault.kv.id +} + +## Manual secrets + +resource "azurerm_key_vault_secret" "application_insights_connection_string" { + name = "app-insight-connection-string" + value = data.azurerm_application_insights.application_insights_italy.connection_string + content_type = "text/plain" + key_vault_id = data.azurerm_key_vault.kv.id +} + + +resource "azurerm_key_vault_secret" "tenant_id" { + name = "tenant-id" + value = data.azurerm_subscription.current.tenant_id + content_type = "text/plain" + key_vault_id = data.azurerm_key_vault.kv.id +} + +# Event Hub + +resource "azurerm_key_vault_secret" "ehub_gpd_ingestion_jaas_config" { + name = "ehub-${var.env_short}-gpd-ingestion-jaas-config" + value = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${azurerm_eventhub_authorization_rule.cdc_connection_string.primary_connection_string}\";" + content_type = "text/plain" + key_vault_id = data.azurerm_key_vault.kv.id +} diff --git a/src/domains/gpdingestion-app/90_pdb.tf b/src/domains/gpdingestion-app/90_pdb.tf new file mode 100644 index 0000000000..68c5c276a6 --- /dev/null +++ b/src/domains/gpdingestion-app/90_pdb.tf @@ -0,0 +1,15 @@ +resource "kubernetes_pod_disruption_budget_v1" "gpd_ingestion" { + + for_each = var.pod_disruption_budgets + + metadata { + namespace = kubernetes_namespace.namespace.metadata[0].name + name = each.key + } + spec { + min_available = each.value.minAvailable + selector { + match_labels = each.value.matchLabels + } + } +} diff --git a/src/domains/gpdingestion-app/99_locals.tf b/src/domains/gpdingestion-app/99_locals.tf new file mode 100644 index 0000000000..9b8dffcb17 --- /dev/null +++ b/src/domains/gpdingestion-app/99_locals.tf @@ -0,0 +1,43 @@ +locals { + product = "${var.prefix}-${var.env_short}" + project_short = "${var.prefix}-${var.env_short}-${var.domain}" + project = "${var.prefix}-${var.env_short}-${var.location_short}-${var.domain}" + + location_short_weu = "weu" + project_short_weu = "${var.prefix}-${var.env_short}-${local.location_short_weu}" + + project_core_itn = "${var.prefix}-${var.env_short}-${var.location_short}-core" + + + monitor_action_group_slack_name = "SlackPagoPA" + monitor_action_group_email_name = "PagoPA" + monitor_action_group_opsgenie_name = "Opsgenie" + monitor_appinsights_name = "${local.product}-appinsights" + monitor_appinsights_italy_name = "${local.project_core_itn}-appinsights" + + vnet_name = "${var.prefix}-${var.env_short}-${var.location_short}-vnet" + vnet_resource_group_name = "${var.prefix}-${var.env_short}-${var.location_short}-vnet-rg" + + aks_name = "${local.product}-${var.location_short}-${var.instance}-aks" + aks_resource_group_name = "${local.product}-${var.location_short}-${var.instance}-aks-rg" + + ingress_hostname = "${var.domain}.itn" + internal_dns_zone_name = "${var.dns_zone_internal_prefix}.${var.external_domain}" + internal_dns_zone_resource_group_name = "${local.product}-vnet-rg" + + pagopa_apim_snet = "${local.product}-apim-snet" + pagopa_vnet_integration = "pagopa-${var.env_short}-vnet-integration" + pagopa_vnet_rg = "pagopa-${var.env_short}-vnet-rg" + + domain_hostname = "${var.dns_zone_prefix}.${local.internal_dns_zone_name}" + + pagopa_apim_name = "${local.product}-apim" + pagopa_apim_rg = "${local.product}-api-rg" + + apim_hostname = "api.${var.apim_dns_zone_prefix}.${var.external_domain}" + hostname = var.env == "prod" ? "${var.domain}.itn.internal.platform.pagopa.it" : "${var.domain}.itn.internal.${var.env}.platform.pagopa.it" + + + evt_hub_location = "${local.location_short_weu}-core" + +} diff --git a/src/domains/gpdingestion-app/99_main.tf b/src/domains/gpdingestion-app/99_main.tf new file mode 100644 index 0000000000..8bf0b91ba8 --- /dev/null +++ b/src/domains/gpdingestion-app/99_main.tf @@ -0,0 +1,49 @@ +terraform { + required_version = ">= 1.6.0" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "<= 3.106.0" + } + azuread = { + source = "hashicorp/azuread" + version = "<= 2.47.0" + } + null = { + source = "hashicorp/null" + version = "<= 3.2.1" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "<= 2.29.0" + } + helm = { + source = "hashicorp/helm" + version = "<= 2.12.1" + } + } + + backend "azurerm" {} +} + +provider "azurerm" { + features { + key_vault { + purge_soft_delete_on_destroy = false + } + } +} + +data "azurerm_subscription" "current" {} + +data "azurerm_client_config" "current" {} + +provider "kubernetes" { + config_path = "${var.k8s_kube_config_path_prefix}/config-${local.aks_name}" +} + +provider "helm" { + kubernetes { + config_path = "${var.k8s_kube_config_path_prefix}/config-${local.aks_name}" + } +} diff --git a/src/domains/gpdingestion-app/99_variables.tf b/src/domains/gpdingestion-app/99_variables.tf new file mode 100644 index 0000000000..e49b5c9cb7 --- /dev/null +++ b/src/domains/gpdingestion-app/99_variables.tf @@ -0,0 +1,158 @@ +# general + +variable "prefix" { + type = string + validation { + condition = ( + length(var.prefix) <= 6 + ) + error_message = "Max length is 6 chars." + } +} + +variable "env" { + type = string +} + +variable "env_short" { + type = string + validation { + condition = ( + length(var.env_short) == 1 + ) + error_message = "Length must be 1 chars." + } +} + +variable "domain" { + type = string + validation { + condition = ( + length(var.domain) <= 12 + ) + error_message = "Max length is 12 chars." + } +} + +variable "location" { + type = string + description = "One of westeurope, northeurope" +} + +variable "location_short" { + type = string + validation { + condition = ( + length(var.location_short) == 3 + ) + error_message = "Length must be 3 chars." + } + description = "One of wue, neu" +} + +variable "location_string" { + type = string + description = "One of West Europe, North Europe" +} + +variable "instance" { + type = string + description = "One of beta, prod01, prod02" +} + +variable "tags" { + type = map(any) + default = { + CreatedBy = "Terraform" + } +} + +### Features flags + +variable "is_feature_enabled" { + type = object({ + gpdingestion = bool + }) + default = { + gpdingestion = false + } +} +### External resources + +variable "monitor_resource_group_name" { + type = string + description = "Monitor resource group name" +} + +variable "log_analytics_workspace_name" { + type = string + description = "Specifies the name of the Log Analytics Workspace." +} + +variable "log_analytics_workspace_resource_group_name" { + type = string + description = "The name of the resource group in which the Log Analytics workspace is located in." +} + +variable "monitor_italy_resource_group_name" { + type = string + description = "Monitor Italy resource group name" +} + +variable "log_analytics_italy_workspace_name" { + type = string + description = "Specifies the name of the Log Analytics Workspace Italy." +} + +variable "log_analytics_italy_workspace_resource_group_name" { + type = string + description = "The name of the resource group in which the Log Analytics workspace Italy is located in." +} + + +### Aks +variable "ingress_load_balancer_ip" { + type = string +} + +variable "k8s_kube_config_path_prefix" { + type = string + default = "~/.kube" +} + +variable "external_domain" { + type = string + default = null + description = "Domain for delegation" +} + +variable "dns_zone_internal_prefix" { + type = string + default = null + description = "The dns subdomain." +} + +variable "apim_dns_zone_prefix" { + type = string + default = null + description = "The dns subdomain for apim." +} + +# DNS + +variable "dns_zone_prefix" { + type = string + default = null + description = "The wallet dns subdomain." +} + +### PDB +variable "pod_disruption_budgets" { + type = map(object({ + name = optional(string, null) + minAvailable = optional(number, null) + matchLabels = optional(map(any), {}) + })) + description = "Pod disruption budget for domain namespace" + default = {} +} diff --git a/src/domains/gpdingestion-app/README.md b/src/domains/gpdingestion-app/README.md new file mode 100644 index 0000000000..51e2d2e81e --- /dev/null +++ b/src/domains/gpdingestion-app/README.md @@ -0,0 +1,106 @@ +# paymentoptions-app + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.6.0 | +| [azuread](#requirement\_azuread) | <= 2.47.0 | +| [azurerm](#requirement\_azurerm) | <= 3.106.0 | +| [helm](#requirement\_helm) | <= 2.12.1 | +| [kubernetes](#requirement\_kubernetes) | <= 2.29.0 | +| [null](#requirement\_null) | <= 3.2.1 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [apim\_api\_pay\_opt\_mock\_api](#module\_apim\_api\_pay\_opt\_mock\_api) | git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_api | v8.18.0 | +| [apim\_payment\_options\_mock\_product](#module\_apim\_payment\_options\_mock\_product) | git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_product | v8.18.0 | +| [apim\_payment\_options\_product](#module\_apim\_payment\_options\_product) | git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_product | v8.18.0 | +| [kubernetes\_service\_account](#module\_kubernetes\_service\_account) | git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_service_account | v8.18.0 | +| [pod\_identity](#module\_pod\_identity) | git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_pod_identity | v8.18.0 | +| [tls\_checker](#module\_tls\_checker) | git::https://github.com/pagopa/terraform-azurerm-v3.git//tls_checker | v8.22.0 | + +## Resources + +| Name | Type | +|------|------| +| [azurerm_api_management_api_version_set.payment_options_mock_api](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/api_management_api_version_set) | resource | +| [azurerm_api_management_subscription.api_config_subkey](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/api_management_subscription) | resource | +| [azurerm_api_management_subscription.forwarder_subkey](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/api_management_subscription) | resource | +| [azurerm_api_management_subscription.service_payment_options_subkey](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/api_management_subscription) | resource | +| [azurerm_key_vault_secret.aks_apiserver_url](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.api_config_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.application_insights_connection_string](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.azure_devops_sa_cacrt](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.azure_devops_sa_token](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.ehub_nodo-dei-pagamenti-verify-ko_jaas_config](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.ehub_nodo_pagamenti_cache_jaas_config](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.ehub_payment-options-re_jaas_config](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.forwarder_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.service_payment_options_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.tenant_id](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_monitor_scheduled_query_rules_alert.pagopa-payment-options-rest-availability-upd](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_scheduled_query_rules_alert) | resource | +| [azurerm_monitor_scheduled_query_rules_alert.pagopa-payment-options-service-responsetime-upd](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_scheduled_query_rules_alert) | resource | +| [azurerm_private_dns_a_record.ingress](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_a_record) | resource | +| [helm_release.cert_mounter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.reloader](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_namespace.namespace](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [kubernetes_namespace.namespace_system](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [kubernetes_pod_disruption_budget_v1.payment_options](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/pod_disruption_budget_v1) | resource | +| [kubernetes_role_binding.deployer_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding) | resource | +| [kubernetes_role_binding.system_deployer_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding) | resource | +| [azurerm_api_management.apim](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/api_management) | data source | +| [azurerm_api_management_product.apim_api_config_product](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/api_management_product) | data source | +| [azurerm_api_management_product.apim_forwarder_product](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/api_management_product) | data source | +| [azurerm_application_insights.application_insights_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/application_insights) | data source | +| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_eventhub_authorization_rule.pagopa_weu_core_evh_ns04_nodo_dei_pagamenti_cache_sync_reader](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | +| [azurerm_eventhub_authorization_rule.pagopa_weu_core_evh_ns04_nodo_dei_pagamenti_verify_ko_writer](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | +| [azurerm_eventhub_authorization_rule.payment_options_re_authorization_rule_writer](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | +| [azurerm_key_vault.kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault) | data source | +| [azurerm_kubernetes_cluster.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_cluster) | data source | +| [azurerm_log_analytics_workspace.log_analytics_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/log_analytics_workspace) | data source | +| [azurerm_monitor_action_group.email](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | +| [azurerm_monitor_action_group.opsgenie](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | +| [azurerm_monitor_action_group.slack](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | +| [azurerm_private_dns_zone.internal](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source | +| [azurerm_resource_group.monitor_italy_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | +| [azurerm_subnet.apim_vnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | +| [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | +| [azurerm_virtual_network.vnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/virtual_network) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [apim\_dns\_zone\_prefix](#input\_apim\_dns\_zone\_prefix) | The dns subdomain for apim. | `string` | `null` | no | +| [dns\_zone\_internal\_prefix](#input\_dns\_zone\_internal\_prefix) | The dns subdomain. | `string` | `null` | no | +| [dns\_zone\_prefix](#input\_dns\_zone\_prefix) | The wallet dns subdomain. | `string` | `null` | no | +| [domain](#input\_domain) | n/a | `string` | n/a | yes | +| [env](#input\_env) | n/a | `string` | n/a | yes | +| [env\_short](#input\_env\_short) | n/a | `string` | n/a | yes | +| [external\_domain](#input\_external\_domain) | Domain for delegation | `string` | `null` | no | +| [ingress\_load\_balancer\_ip](#input\_ingress\_load\_balancer\_ip) | ## Aks | `string` | n/a | yes | +| [instance](#input\_instance) | One of beta, prod01, prod02 | `string` | n/a | yes | +| [is\_feature\_enabled](#input\_is\_feature\_enabled) | n/a | 
object({
    paymentoptions      = bool
    paymentoptions_mock = bool
  })
| 
{
  "paymentoptions": false,
  "paymentoptions_mock": false
}
| no | +| [k8s\_kube\_config\_path\_prefix](#input\_k8s\_kube\_config\_path\_prefix) | n/a | `string` | `"~/.kube"` | no | +| [location](#input\_location) | One of westeurope, northeurope | `string` | n/a | yes | +| [location\_short](#input\_location\_short) | One of wue, neu | `string` | n/a | yes | +| [location\_string](#input\_location\_string) | One of West Europe, North Europe | `string` | n/a | yes | +| [log\_analytics\_italy\_workspace\_name](#input\_log\_analytics\_italy\_workspace\_name) | Specifies the name of the Log Analytics Workspace Italy. | `string` | n/a | yes | +| [log\_analytics\_italy\_workspace\_resource\_group\_name](#input\_log\_analytics\_italy\_workspace\_resource\_group\_name) | The name of the resource group in which the Log Analytics workspace Italy is located in. | `string` | n/a | yes | +| [log\_analytics\_workspace\_name](#input\_log\_analytics\_workspace\_name) | Specifies the name of the Log Analytics Workspace. | `string` | n/a | yes | +| [log\_analytics\_workspace\_resource\_group\_name](#input\_log\_analytics\_workspace\_resource\_group\_name) | The name of the resource group in which the Log Analytics workspace is located in. | `string` | n/a | yes | +| [monitor\_italy\_resource\_group\_name](#input\_monitor\_italy\_resource\_group\_name) | Monitor Italy resource group name | `string` | n/a | yes | +| [monitor\_resource\_group\_name](#input\_monitor\_resource\_group\_name) | Monitor resource group name | `string` | n/a | yes | +| [pod\_disruption\_budgets](#input\_pod\_disruption\_budgets) | Pod disruption budget for domain namespace | 
map(object({
    name         = optional(string, null)
    minAvailable = optional(number, null)
    matchLabels  = optional(map(any), {})
  }))
| `{}` | no | +| [prefix](#input\_prefix) | n/a | `string` | n/a | yes | +| [tags](#input\_tags) | n/a | `map(any)` | 
{
  "CreatedBy": "Terraform"
}
| no | + +## Outputs + +No outputs. + diff --git a/src/domains/gpdingestion-app/env/itn-dev/backend.ini b/src/domains/gpdingestion-app/env/itn-dev/backend.ini new file mode 100644 index 0000000000..f3ea2d530c --- /dev/null +++ b/src/domains/gpdingestion-app/env/itn-dev/backend.ini @@ -0,0 +1 @@ +subscription=DEV-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-app/env/itn-dev/backend.tfvars b/src/domains/gpdingestion-app/env/itn-dev/backend.tfvars new file mode 100644 index 0000000000..127a949568 --- /dev/null +++ b/src/domains/gpdingestion-app/env/itn-dev/backend.tfvars @@ -0,0 +1,4 @@ +resource_group_name = "terraform-state-rg" +storage_account_name = "tfinfdevpagopa" +container_name = "terraform-state" +key = "gpdingestion-app-dev.terraform.tfstate" diff --git a/src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars b/src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars new file mode 100644 index 0000000000..880d882c52 --- /dev/null +++ b/src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars @@ -0,0 +1,38 @@ +prefix = "pagopa" +env_short = "d" +env = "dev" +domain = "gpdingestion" +location = "italynorth" +location_short = "itn" +location_string = "Italy North" +instance = "dev" + +tags = { + CreatedBy = "Terraform" + Environment = "Dev" + Owner = "pagoPA" + Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-app" + CostCenter = "TS310 - PAGAMENTI & SERVIZI" +} + +### External resources + +monitor_italy_resource_group_name = "pagopa-d-itn-core-monitor-rg" +log_analytics_italy_workspace_name = "pagopa-d-itn-core-law" +log_analytics_italy_workspace_resource_group_name = "pagopa-d-itn-core-monitor-rg" + +monitor_resource_group_name = "pagopa-d-monitor-rg" +log_analytics_workspace_name = "pagopa-d-law" +log_analytics_workspace_resource_group_name = "pagopa-d-monitor-rg" + +external_domain = "pagopa.it" +dns_zone_internal_prefix = "internal.dev.platform" +dns_zone_prefix = "gpdingestion.itn" +apim_dns_zone_prefix = "dev.platform" +### Aks + +ingress_load_balancer_ip = "10.3.2.250" + +is_feature_enabled = { + gpdingestion = true +} diff --git a/src/domains/gpdingestion-app/env/itn-prod/backend.ini b/src/domains/gpdingestion-app/env/itn-prod/backend.ini new file mode 100644 index 0000000000..ddda4bb50f --- /dev/null +++ b/src/domains/gpdingestion-app/env/itn-prod/backend.ini @@ -0,0 +1 @@ +subscription=prod-pagoPA diff --git a/src/domains/gpdingestion-app/env/itn-prod/backend.tfvars b/src/domains/gpdingestion-app/env/itn-prod/backend.tfvars new file mode 100644 index 0000000000..41b804626c --- /dev/null +++ b/src/domains/gpdingestion-app/env/itn-prod/backend.tfvars @@ -0,0 +1,4 @@ +resource_group_name = "terraform-state-rg" +storage_account_name = "tfinfprodpagopa" +container_name = "terraform-state" +key = "gpdingestion-app-prod.terraform.tfstate" diff --git a/src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars b/src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars new file mode 100644 index 0000000000..877f59b559 --- /dev/null +++ b/src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars @@ -0,0 +1,47 @@ +prefix = "pagopa" +env_short = "p" +env = "prod" +domain = "gpdingestion" +location = "italynorth" +location_short = "itn" +location_string = "Italy North" +instance = "prod" + +tags = { + CreatedBy = "Terraform" + Environment = "prod" + Owner = "pagoPA" + Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-app" + CostCenter = "TS310 - PAGAMENTI & SERVIZI" +} + +### External resources + +monitor_italy_resource_group_name = "pagopa-p-itn-core-monitor-rg" +log_analytics_italy_workspace_name = "pagopa-p-itn-core-law" +log_analytics_italy_workspace_resource_group_name = "pagopa-p-itn-core-monitor-rg" + +monitor_resource_group_name = "pagopa-p-monitor-rg" +log_analytics_workspace_name = "pagopa-p-law" +log_analytics_workspace_resource_group_name = "pagopa-p-monitor-rg" + +external_domain = "pagopa.it" +dns_zone_internal_prefix = "internal.platform" +dns_zone_prefix = "gpdingestion.itn" +apim_dns_zone_prefix = "platform" +### Aks + +ingress_load_balancer_ip = "10.3.2.250" + +is_feature_enabled = { + gpdingestion = true +} + +pod_disruption_budgets = { + "gpd-ingestion-manager" = { + minAvailable = 2 + matchLabels = { + "app.kubernetes.io/instance" = "gpd-ingestion-manager" + } + }, +} diff --git a/src/domains/gpdingestion-app/env/itn-uat/backend.ini b/src/domains/gpdingestion-app/env/itn-uat/backend.ini new file mode 100644 index 0000000000..1759a0ca0d --- /dev/null +++ b/src/domains/gpdingestion-app/env/itn-uat/backend.ini @@ -0,0 +1 @@ +subscription=UAT-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-app/env/itn-uat/backend.tfvars b/src/domains/gpdingestion-app/env/itn-uat/backend.tfvars new file mode 100644 index 0000000000..b7b0918896 --- /dev/null +++ b/src/domains/gpdingestion-app/env/itn-uat/backend.tfvars @@ -0,0 +1,4 @@ +resource_group_name = "terraform-state-rg" +storage_account_name = "tfinfuatpagopa" +container_name = "terraform-state" +key = "gpdingestion-app-uat.terraform.tfstate" diff --git a/src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars b/src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars new file mode 100644 index 0000000000..2ece4eb2a7 --- /dev/null +++ b/src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars @@ -0,0 +1,38 @@ +prefix = "pagopa" +env_short = "u" +env = "uat" +domain = "gpdingestion" +location = "italynorth" +location_short = "itn" +location_string = "Italy North" +instance = "uat" + +tags = { + CreatedBy = "Terraform" + Environment = "Uat" + Owner = "pagoPA" + Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-app" + CostCenter = "TS310 - PAGAMENTI & SERVIZI" +} + +### External resources + +monitor_italy_resource_group_name = "pagopa-u-itn-core-monitor-rg" +log_analytics_italy_workspace_name = "pagopa-u-itn-core-law" +log_analytics_italy_workspace_resource_group_name = "pagopa-u-itn-core-monitor-rg" + +monitor_resource_group_name = "pagopa-u-monitor-rg" +log_analytics_workspace_name = "pagopa-u-law" +log_analytics_workspace_resource_group_name = "pagopa-u-monitor-rg" + +external_domain = "pagopa.it" +dns_zone_internal_prefix = "internal.uat.platform" +dns_zone_prefix = "gpdingestion.itn" +apim_dns_zone_prefix = "uat.platform" +### Aks + +ingress_load_balancer_ip = "10.3.2.250" + +is_feature_enabled = { + gpdingestion = true +} diff --git a/src/domains/gpdingestion-app/helm/cert-mounter.yaml.tpl b/src/domains/gpdingestion-app/helm/cert-mounter.yaml.tpl new file mode 100644 index 0000000000..73ee05d737 --- /dev/null +++ b/src/domains/gpdingestion-app/helm/cert-mounter.yaml.tpl @@ -0,0 +1,13 @@ +namespace: ${NAMESPACE} +nameOverride: "" +fullnameOverride: "" + +deployment: + create: true + +kvCertificatesName: + - ${CERTIFICATE_NAME} + +keyvault: + name: "${KV_NAME}" + tenantId: "7788edaf-0346-4068-9d79-c868aed15b3d" diff --git a/src/domains/gpdingestion-app/terraform.sh b/src/domains/gpdingestion-app/terraform.sh new file mode 100755 index 0000000000..047a7512d0 --- /dev/null +++ b/src/domains/gpdingestion-app/terraform.sh @@ -0,0 +1,324 @@ +#!/bin/bash +############################################################ +# Terraform script for managing infrastructure on Azure +# Fingerprint: d2hhdHlvdXdhbnQ/Cg== +############################################################ +# Global variables +# Version format x.y accepted +vers="1.11" +script_name=$(basename "$0") +git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/azure/${script_name}" +tmp_file="${script_name}.new" +# Check if the third parameter exists and is a file +if [ -n "$3" ] && [ -f "$3" ]; then + FILE_ACTION=true +else + FILE_ACTION=false +fi + +# Define functions +function clean_environment() { + rm -rf .terraform + rm tfplan 2>/dev/null + echo "cleaned!" +} + +function download_tool() { + #default value + cpu_type="intel" + os_type=$(uname) + + # only on MacOS + if [ "$os_type" == "Darwin" ]; then + cpu_brand=$(sysctl -n machdep.cpu.brand_string) + if grep -q -i "intel" <<< "$cpu_brand"; then + cpu_type="intel" + else + cpu_type="arm" + fi + fi + + echo $cpu_type + tool=$1 + git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/golang/${tool}_${cpu_type}" + if ! command -v $tool &> /dev/null; then + if ! curl -sL "$git_repo" -o "$tool"; then + echo "Error downloading ${tool}" + return 1 + else + chmod +x $tool + echo "${tool} downloaded! Please note this tool WON'T be copied in your **/bin folder for safety reasons. +You need to do it yourself!" + read -p "Press enter to continue" + + + fi + fi +} + +function extract_resources() { + TF_FILE=$1 + ENV=$2 + TARGETS="" + + # Check if the file exists + if [ ! -f "$TF_FILE" ]; then + echo "File $TF_FILE does not exist." + exit 1 + fi + + # Check if the directory exists + if [ ! -d "./env/$ENV" ]; then + echo "Directory ./env/$ENV does not exist." + exit 1 + fi + + TMP_FILE=$(mktemp) + grep -E '^resource|^module' $TF_FILE > $TMP_FILE + + while read -r line ; do + TYPE=$(echo $line | cut -d '"' -f 1 | tr -d ' ') + if [ "$TYPE" == "module" ]; then + NAME=$(echo $line | cut -d '"' -f 2) + TARGETS+=" -target=\"$TYPE.$NAME\"" + else + NAME1=$(echo $line | cut -d '"' -f 2) + NAME2=$(echo $line | cut -d '"' -f 4) + TARGETS+=" -target=\"$NAME1.$NAME2\"" + fi + done < $TMP_FILE + + rm $TMP_FILE + + echo "./terraform.sh $action $ENV $TARGETS" +} + +function help_usage() { + echo "terraform.sh Version ${vers}" + echo + echo "Usage: ./script.sh [ACTION] [ENV] [OTHER OPTIONS]" + echo "es. ACTION: init, apply, plan, etc." + echo "es. ENV: dev, uat, prod, etc." + echo + echo "Available actions:" + echo " clean Remove .terraform* folders and tfplan files" + echo " help This help" + echo " list List every environment available" + echo " update Update this script if possible" + echo " summ Generate summary of Terraform plan" + echo " tflist Generate an improved output of terraform state list" + echo " tlock Generate or update the dependency lock file" + echo " * any terraform option" +} + +function init_terraform() { + if [ -n "$env" ]; then + terraform init -reconfigure -backend-config="./env/$env/backend.tfvars" + else + echo "ERROR: no env configured!" + exit 1 + fi +} + +function list_env() { + # Check if env directory exists + if [ ! -d "./env" ]; then + echo "No environment directory found" + exit 1 + fi + + # List subdirectories under env directory + env_list=$(ls -d ./env/*/ 2>/dev/null) + + # Check if there are any subdirectories + if [ -z "$env_list" ]; then + echo "No environments found" + exit 1 + fi + + # Print the list of environments + echo "Available environments:" + for env in $env_list; do + env_name=$(echo "$env" | sed 's#./env/##;s#/##') + echo "- $env_name" + done +} + +function other_actions() { + if [ -n "$env" ] && [ -n "$action" ]; then + terraform "$action" -var-file="./env/$env/terraform.tfvars" -compact-warnings $other + else + echo "ERROR: no env or action configured!" + exit 1 + fi +} + +function state_output_taint_actions() { + if [ "$action" == "tflist" ]; then + # If 'tflist' is not installed globally and there is no 'tflist' file in the current directory, + # attempt to download the 'tflist' tool + if ! command -v tflist &> /dev/null && [ ! -f "tflist" ]; then + download_tool "tflist" + if [ $? -ne 0 ]; then + echo "Error: Failed to download tflist!!" + exit 1 + else + echo "tflist downloaded!" + fi + fi + if command -v tflist &> /dev/null; then + terraform state list | tflist + else + terraform state list | ./tflist + fi + else + terraform $action $other + fi +} + + +function parse_tfplan_option() { + # Create an array to contain arguments that do not start with '-tfplan=' + local other_args=() + + # Loop over all arguments + for arg in "$@"; do + # If the argument starts with '-tfplan=', extract the file name + if [[ "$arg" =~ ^-tfplan= ]]; then + echo "${arg#*=}" + else + # If the argument does not start with '-tfplan=', add it to the other_args array + other_args+=("$arg") + fi + done + + # Print all arguments in other_args separated by spaces + echo "${other_args[@]}" +} + +function tfsummary() { + local plan_file + plan_file=$(parse_tfplan_option "$@") + if [ -z "$plan_file" ]; then + plan_file="tfplan" + fi + action="plan" + other="-out=${plan_file}" + other_actions + if [ -n "$(command -v tf-summarize)" ]; then + tf-summarize -tree "${plan_file}" + else + echo "tf-summarize is not installed" + fi + if [ "$plan_file" == "tfplan" ]; then + rm $plan_file + fi +} + +function update_script() { + # Check if the repository was cloned successfully + if ! curl -sL "$git_repo" -o "$tmp_file"; then + echo "Error cloning the repository" + rm "$tmp_file" 2>/dev/null + return 1 + fi + + # Check if a newer version exists + remote_vers=$(sed -n '8s/vers="\(.*\)"/\1/p' "$tmp_file") + if [ "$(printf '%s\n' "$vers" "$remote_vers" | sort -V | tail -n 1)" == "$vers" ]; then + echo "The local script version is equal to or newer than the remote version." + rm "$tmp_file" 2>/dev/null + return 0 + fi + + # Check the fingerprint + local_fingerprint=$(sed -n '4p' "$0") + remote_fingerprint=$(sed -n '4p' "$tmp_file") + + if [ "$local_fingerprint" != "$remote_fingerprint" ]; then + echo "The local and remote file fingerprints do not match." + rm "$tmp_file" 2>/dev/null + return 0 + fi + + # Show the current and available versions to the user + echo "Current script version: $vers" + echo "Available script version: $remote_vers" + + # Ask the user if they want to update the script + read -rp "Do you want to update the script to version $remote_vers? (y/n): " answer + + if [ "$answer" == "y" ] || [ "$answer" == "Y" ]; then + # Replace the local script with the updated version + cp "$tmp_file" "$script_name" + chmod +x "$script_name" + rm "$tmp_file" 2>/dev/null + + echo "Script successfully updated to version $remote_vers" + else + echo "Update canceled by the user" + fi + + rm "$tmp_file" 2>/dev/null +} + +# Check arguments number +if [ "$#" -lt 1 ]; then + help_usage + exit 0 +fi + +# Parse arguments +action=$1 +env=$2 +filetf=$3 +shift 2 +other=$@ + +if [ -n "$env" ]; then + # shellcheck source=/dev/null + source "./env/$env/backend.ini" + if [ -z "$(command -v az)" ]; then + echo "az not found, cannot proceed" + exit 1 + fi + az account set -s "${subscription}" +fi + +# Call appropriate function based on action +case $action in + clean) + clean_environment + ;; + ?|help|-h) + help_usage + ;; + init) + init_terraform "$other" + ;; + list) + list_env + ;; + output|state|taint|tflist) + init_terraform + state_output_taint_actions $other + ;; + summ) + init_terraform + tfsummary "$other" + ;; + tlock) + terraform providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=darwin_arm64 -platform=linux_amd64 + ;; + update) + update_script + ;; + *) + if [ "$FILE_ACTION" = true ]; then + extract_resources "$filetf" "$env" + else + init_terraform + other_actions "$other" + fi + ;; +esac diff --git a/src/domains/gpdingestion-common/.terraform.lock.hcl b/src/domains/gpdingestion-common/.terraform.lock.hcl new file mode 100644 index 0000000000..c1bc80ffb2 --- /dev/null +++ b/src/domains/gpdingestion-common/.terraform.lock.hcl @@ -0,0 +1,65 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "2.47.0" + constraints = "<= 2.47.0" + hashes = [ + "h1:g8+gBFM4QVOEQFqAEs5pR6iXpbGvgPvcEi1evHwziyw=", + "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", + "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", + "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", + "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", + "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", + "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", + "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", + "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", + "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", + "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", + "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.106.0" + constraints = "~> 3.30, <= 3.106.0" + hashes = [ + "h1:6t9Nz9tYAR9BfHZ8yc56m+GKRl0nriwjQ5DyA0/TnCs=", + "h1:Mxe1/I27IZK3BP6cm84Gt0+7PXd2EDaDUMxuljm/rUA=", + "zh:07980d6fdc40c0adb670c8413a5c667917d6dbb51fcedc467c35d64c2f3a1f47", + "zh:2e6e8491b1f089644b0d23f8da83398f1e10cf5a62b16efcef2b5454fe923038", + "zh:450dbd72821c5619cc3bcdc20fdd0e29515147e44b733f9c79d3a75851810055", + "zh:5e234c0a2f3c9677ea72b2a6e6ca90defb99fab29ae565f5d1f70728ba4ba78f", + "zh:83fd042ece6977429d79affd03d6ce963d2f122604dbf15a1abf203d7a7bbc8a", + "zh:93027e1f66b3bf83398d572d4e6f6e7777330c78c54da3226dadd50fd868ada9", + "zh:ae3d1dd66140c303df97d93c47a60f16735ce17cf156f45475dcee4a7360af5b", + "zh:daf9d2eb89e785458a76b88bf2ef0696c472094c77cc9cff3b3ea4b885c5a482", + "zh:dd46370141651e6549da6d85e25c7a6770c47581bbaaa27eda2886d41d849747", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f77405c0d8f6e0d93d9da83256b3b02c164bad4c791ed9604310ff02ae086ad1", + "zh:ffa769147bda833aef8802e3a391bd175ec749862764d61cbdaa8200d5b8f893", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.2" + constraints = "<= 3.2.2" + hashes = [ + "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", + "h1:vWAsYRd7MjYr3adj8BVKRohVfHpWQdvkIwUQ2Jf5FVM=", + "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", + "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", + "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", + "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", + "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", + "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", + "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", + "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", + "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", + "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", + ] +} diff --git a/src/domains/gpdingestion-common/00_data.tf b/src/domains/gpdingestion-common/00_data.tf new file mode 100644 index 0000000000..453409f78e --- /dev/null +++ b/src/domains/gpdingestion-common/00_data.tf @@ -0,0 +1,4 @@ +data "azurerm_key_vault" "kv" { + name = "${local.project}-kv" + resource_group_name = "${local.project}-sec-rg" +} diff --git a/src/domains/gpdingestion-common/00_monitor.tf b/src/domains/gpdingestion-common/00_monitor.tf new file mode 100644 index 0000000000..3be0e0c27b --- /dev/null +++ b/src/domains/gpdingestion-common/00_monitor.tf @@ -0,0 +1,45 @@ +# +# 🇮🇹 Monitor Italy +# +data "azurerm_resource_group" "monitor_italy_rg" { + name = var.monitor_italy_resource_group_name +} + +data "azurerm_log_analytics_workspace" "log_analytics_italy" { + name = var.log_analytics_italy_workspace_name + resource_group_name = var.log_analytics_italy_workspace_resource_group_name +} + +data "azurerm_application_insights" "application_insights_italy" { + name = local.monitor_appinsights_italy_name + resource_group_name = data.azurerm_resource_group.monitor_italy_rg.name +} + +# ### 🇪🇺 +# data "azurerm_resource_group" "monitor_rg" { +# name = var.monitor_resource_group_name +# } +# +# data "azurerm_log_analytics_workspace" "log_analytics" { +# name = var.log_analytics_workspace_name +# resource_group_name = var.log_analytics_workspace_resource_group_name +# } +# +# data "azurerm_application_insights" "application_insights" { +# name = local.monitor_appinsights_name +# resource_group_name = data.azurerm_resource_group.monitor_rg.name +# } + +# +# Action Groups +# +data "azurerm_monitor_action_group" "slack" { + resource_group_name = var.monitor_resource_group_name + name = local.monitor_action_group_slack_name +} + +data "azurerm_monitor_action_group" "email" { + resource_group_name = var.monitor_resource_group_name + name = local.monitor_action_group_email_name +} + diff --git a/src/domains/gpdingestion-common/00_network.tf b/src/domains/gpdingestion-common/00_network.tf new file mode 100644 index 0000000000..73fad2990a --- /dev/null +++ b/src/domains/gpdingestion-common/00_network.tf @@ -0,0 +1,37 @@ +data "azurerm_virtual_network" "vnet_italy" { + name = local.vnet_italy_name + resource_group_name = local.vnet_italy_resource_group_name +} + +data "azurerm_resource_group" "rg_vnet_italy" { + name = local.vnet_italy_resource_group_name +} + +# +# Subnets +# +data "azurerm_subnet" "aks_subnet" { + name = local.aks_subnet_name + virtual_network_name = local.vnet_italy_name + resource_group_name = local.vnet_italy_resource_group_name +} + +# +# Private DNS Zones +# +data "azurerm_private_dns_zone" "internal" { + name = local.internal_dns_zone_name + resource_group_name = local.internal_dns_zone_resource_group_name +} + +# +# Eventhub +# +data "azurerm_private_dns_zone" "eventhub" { + name = "privatelink.servicebus.windows.net" + resource_group_name = local.msg_resource_group_name +} + +data "azurerm_resource_group" "rg_event_private_dns_zone" { + name = local.msg_resource_group_name +} diff --git a/src/domains/gpdingestion-common/01_network.tf b/src/domains/gpdingestion-common/01_network.tf new file mode 100644 index 0000000000..7a80e7444b --- /dev/null +++ b/src/domains/gpdingestion-common/01_network.tf @@ -0,0 +1,14 @@ +resource "azurerm_private_dns_a_record" "ingress" { + name = local.ingress_hostname + zone_name = data.azurerm_private_dns_zone.internal.name + resource_group_name = local.internal_dns_zone_resource_group_name + ttl = 3600 + records = [var.ingress_load_balancer_ip] +} + +resource "azurerm_subnet" "eventhub_italy" { + name = "${local.project}-eventhub-snet" + resource_group_name = data.azurerm_resource_group.rg_vnet_italy.name + virtual_network_name = data.azurerm_virtual_network.vnet_italy.name + address_prefixes = var.cidr_gpdingestion_eventhub_italy +} diff --git a/src/domains/gpdingestion-common/03_eventhub.tf b/src/domains/gpdingestion-common/03_eventhub.tf new file mode 100644 index 0000000000..98cc499115 --- /dev/null +++ b/src/domains/gpdingestion-common/03_eventhub.tf @@ -0,0 +1,63 @@ +resource "azurerm_resource_group" "eventhub_ita_rg" { + name = local.eventhub_resource_group_name + location = var.location + + tags = var.tags +} + +module "eventhub_namespace" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub?ref=v8.22.0" + name = "${local.project}-evh" + location = var.location + resource_group_name = azurerm_resource_group.eventhub_ita_rg.name + auto_inflate_enabled = var.ehns_auto_inflate_enabled + sku = var.ehns_sku_name + capacity = var.ehns_capacity + maximum_throughput_units = var.ehns_maximum_throughput_units + #zone_redundat is always true + + virtual_network_ids = [data.azurerm_virtual_network.vnet_italy.id] + private_endpoint_subnet_id = azurerm_subnet.eventhub_italy.id + public_network_access_enabled = var.ehns_public_network_access + private_endpoint_created = var.ehns_private_endpoint_is_present + + private_endpoint_resource_group_name = azurerm_resource_group.eventhub_ita_rg.name + + private_dns_zones = { + id = [data.azurerm_private_dns_zone.eventhub.id] + name = [data.azurerm_private_dns_zone.eventhub.name] + resource_group_name = data.azurerm_resource_group.rg_event_private_dns_zone.name + } + + private_dns_zone_record_A_name = "${var.domain}.${var.location_short}" + + action = [ + { + action_group_id = data.azurerm_monitor_action_group.slack.id + webhook_properties = null + }, + { + action_group_id = data.azurerm_monitor_action_group.email.id + webhook_properties = null + } + ] + + metric_alerts_create = var.ehns_alerts_enabled + metric_alerts = var.ehns_metric_alerts + + tags = var.tags +} + +# +# CONFIGURATION +# +module "eventhub_gpdingestion_configuration" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration?ref=v8.22.0" + count = var.is_feature_enabled.eventhub ? 1 : 0 + + event_hub_namespace_name = module.eventhub_namespace.name + event_hub_namespace_resource_group_name = azurerm_resource_group.eventhub_ita_rg.name + + eventhubs = [] +} + diff --git a/src/domains/gpdingestion-common/10_github_identity.tf b/src/domains/gpdingestion-common/10_github_identity.tf new file mode 100644 index 0000000000..4f63c95a2e --- /dev/null +++ b/src/domains/gpdingestion-common/10_github_identity.tf @@ -0,0 +1,207 @@ +data "azurerm_resource_group" "identity_rg" { + name = "${local.product}-identity-rg" +} + +data "azurerm_kubernetes_cluster" "aks" { + name = "${local.product}-${var.location_short}-${var.instance}-aks" + resource_group_name = "${local.product}-${var.location_short}-${var.instance}-aks-rg" +} + +data "azurerm_key_vault" "key_vault" { + name = "${local.product}-${var.location_short}-${var.domain}-kv" + resource_group_name = "${local.product}-${var.location_short}-${var.domain}-sec-rg" +} + +# repos must be lower than 20 items +locals { + repos_01 = [ + "pagopa-gpd-ingestion-manager", + ] + + federations_01 = [ + for repo in local.repos_01 : { + repository = repo + subject = var.env + } + ] + + federations_01_pr = [ + for repo in local.repos_01 : { + repository = repo + subject = "pull_request" + } + ] + + federations_01_ref = [ + for repo in local.repos_01 : { + repository = repo + credentials_scope = "ref" + subject = "refs/heads/main" + } + ] + + + # to avoid subscription Contributor -> https://github.com/microsoft/azure-container-apps/issues/35 + environment_cd_roles = { + subscription = [ + "Contributor", + ] + resource_groups = { + "${local.product}-${var.location_short}-${var.domain}-sec-rg" = [ + "Key Vault Reader" + ], + "${local.product}-${var.location_short}-${var.env}-aks-rg" = [ + "Contributor" + ], + } + } +} + +# create a module for each 20 repos +module "identity_cd_01" { + source = "github.com/pagopa/terraform-azurerm-v3//github_federated_identity?ref=v8.22.0" + # pagopa---github--identity + prefix = var.prefix + env_short = var.env_short + domain = "${var.domain}-01" + + identity_role = "cd" + + github_federations = local.federations_01 + + cd_rbac_roles = { + subscription_roles = local.environment_cd_roles.subscription + resource_groups = local.environment_cd_roles.resource_groups + } + + tags = var.tags + + depends_on = [ + data.azurerm_resource_group.identity_rg + ] +} + +resource "azurerm_key_vault_access_policy" "gha_iac_managed_identities" { + key_vault_id = data.azurerm_key_vault.key_vault.id + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = module.identity_cd_01.identity_principal_id + + secret_permissions = ["Get", "List", "Set", ] + + certificate_permissions = ["SetIssuers", "DeleteIssuers", "Purge", "List", "Get"] + key_permissions = [ + "Get", "List", "Update", "Create", "Import", "Delete", "Encrypt", "Decrypt", "GetRotationPolicy" + ] + + storage_permissions = [] +} + +resource "null_resource" "github_runner_app_permissions_to_namespace_cd_01" { + triggers = { + aks_id = data.azurerm_kubernetes_cluster.aks.id + service_principal_id = module.identity_cd_01.identity_client_id + namespace = var.domain + version = "v2" + } + + provisioner "local-exec" { + command = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.6 | +| [azuread](#requirement\_azuread) | <= 2.47.0 | +| [azurerm](#requirement\_azurerm) | <= 3.106.0 | +| [null](#requirement\_null) | <= 3.2.2 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eventhub\_namespace](#module\_eventhub\_namespace) | git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub | v8.22.0 | +| [eventhub\_paymentoptions\_configuration](#module\_eventhub\_paymentoptions\_configuration) | git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration | v8.22.0 | +| [identity\_cd\_01](#module\_identity\_cd\_01) | github.com/pagopa/terraform-azurerm-v3//github_federated_identity | v8.22.0 | +| [identity\_pr\_01](#module\_identity\_pr\_01) | github.com/pagopa/terraform-azurerm-v3//github_federated_identity | v8.22.0 | +| [identity\_ref\_01](#module\_identity\_ref\_01) | github.com/pagopa/terraform-azurerm-v3//github_federated_identity | v8.36.1 | + +## Resources + +| Name | Type | +|------|------| +| [azurerm_key_vault_access_policy.gha_iac_managed_identities](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | +| [azurerm_key_vault_access_policy.gha_pr_iac_managed_identities](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | +| [azurerm_key_vault_access_policy.gha_ref_iac_managed_identities](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | +| [azurerm_private_dns_a_record.ingress](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_a_record) | resource | +| [azurerm_resource_group.eventhub_ita_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | +| [azurerm_subnet.eventhub_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | +| [null_resource.github_runner_app_permissions_to_namespace_cd_01](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [azurerm_application_insights.application_insights_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/application_insights) | data source | +| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_key_vault.key_vault](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault) | data source | +| [azurerm_key_vault.kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault) | data source | +| [azurerm_kubernetes_cluster.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_cluster) | data source | +| [azurerm_log_analytics_workspace.log_analytics_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/log_analytics_workspace) | data source | +| [azurerm_monitor_action_group.email](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | +| [azurerm_monitor_action_group.slack](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | +| [azurerm_private_dns_zone.eventhub](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source | +| [azurerm_private_dns_zone.internal](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source | +| [azurerm_resource_group.identity_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | +| [azurerm_resource_group.monitor_italy_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | +| [azurerm_resource_group.rg_event_private_dns_zone](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | +| [azurerm_resource_group.rg_vnet_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | +| [azurerm_subnet.aks_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | +| [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | +| [azurerm_virtual_network.vnet_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/virtual_network) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [cidr\_paymentoptions\_eventhub\_italy](#input\_cidr\_paymentoptions\_eventhub\_italy) | Address prefixes for all evh accounts in italy. | `list(string)` | n/a | yes | +| [dns\_zone\_internal\_prefix](#input\_dns\_zone\_internal\_prefix) | The dns subdomain. | `string` | `null` | no | +| [dns\_zone\_platform](#input\_dns\_zone\_platform) | The platform dns subdomain. | `string` | `null` | no | +| [dns\_zone\_prefix](#input\_dns\_zone\_prefix) | The wallet dns subdomain. | `string` | `null` | no | +| [domain](#input\_domain) | n/a | `string` | n/a | yes | +| [ehns\_alerts\_enabled](#input\_ehns\_alerts\_enabled) | Event hub alerts enabled? | `bool` | n/a | yes | +| [ehns\_auto\_inflate\_enabled](#input\_ehns\_auto\_inflate\_enabled) | Is Auto Inflate enabled for the EventHub Namespace? | `bool` | n/a | yes | +| [ehns\_capacity](#input\_ehns\_capacity) | Specifies the Capacity / Throughput Units for a Standard SKU namespace. | `number` | n/a | yes | +| [ehns\_maximum\_throughput\_units](#input\_ehns\_maximum\_throughput\_units) | Specifies the maximum number of throughput units when Auto Inflate is Enabled | `number` | n/a | yes | +| [ehns\_metric\_alerts](#input\_ehns\_metric\_alerts) | Map of name = criteria objects | 
map(object({
    # criteria.*.aggregation to be one of [Average Count Minimum Maximum Total]
    aggregation = string
    metric_name = string
    description = string
    # criteria.0.operator to be one of [Equals NotEquals GreaterThan GreaterThanOrEqual LessThan LessThanOrEqual]
    operator  = string
    threshold = number
    # Possible values are PT1M, PT5M, PT15M, PT30M and PT1H
    frequency = string
    # Possible values are PT1M, PT5M, PT15M, PT30M, PT1H, PT6H, PT12H and P1D.
    window_size = string

    dimension = list(object(
      {
        name     = string
        operator = string
        values   = list(string)
      }
    ))
  }))
| `{}` | no | +| [ehns\_private\_endpoint\_is\_present](#input\_ehns\_private\_endpoint\_is\_present) | (Required) create private endpoint to the event hubs | `bool` | n/a | yes | +| [ehns\_public\_network\_access](#input\_ehns\_public\_network\_access) | (Required) enables public network access to the event hubs | `bool` | n/a | yes | +| [ehns\_sku\_name](#input\_ehns\_sku\_name) | Defines which tier to use. | `string` | n/a | yes | +| [ehns\_zone\_redundant](#input\_ehns\_zone\_redundant) | Specifies if the EventHub Namespace should be Zone Redundant (created across Availability Zones). | `bool` | n/a | yes | +| [env](#input\_env) | n/a | `string` | n/a | yes | +| [env\_short](#input\_env\_short) | n/a | `string` | n/a | yes | +| [external\_domain](#input\_external\_domain) | Domain for delegation | `string` | `null` | no | +| [ingress\_load\_balancer\_ip](#input\_ingress\_load\_balancer\_ip) | n/a | `string` | n/a | yes | +| [instance](#input\_instance) | One of beta, prod01, prod02 | `string` | n/a | yes | +| [is\_feature\_enabled](#input\_is\_feature\_enabled) | n/a | 
object({
    eventhub = bool
  })
| 
{
  "eventhub": false
}
| no | +| [location](#input\_location) | One of westeurope, northeurope | `string` | n/a | yes | +| [location\_short](#input\_location\_short) | One of wue, neu | `string` | `"itn"` | no | +| [log\_analytics\_italy\_workspace\_name](#input\_log\_analytics\_italy\_workspace\_name) | Specifies the name of the Log Analytics Workspace Italy. | `string` | n/a | yes | +| [log\_analytics\_italy\_workspace\_resource\_group\_name](#input\_log\_analytics\_italy\_workspace\_resource\_group\_name) | The name of the resource group in which the Log Analytics workspace Italy is located in. | `string` | n/a | yes | +| [log\_analytics\_workspace\_name](#input\_log\_analytics\_workspace\_name) | Specifies the name of the Log Analytics Workspace. | `string` | n/a | yes | +| [log\_analytics\_workspace\_resource\_group\_name](#input\_log\_analytics\_workspace\_resource\_group\_name) | The name of the resource group in which the Log Analytics workspace is located in. | `string` | n/a | yes | +| [monitor\_italy\_resource\_group\_name](#input\_monitor\_italy\_resource\_group\_name) | Monitor Italy resource group name | `string` | n/a | yes | +| [monitor\_resource\_group\_name](#input\_monitor\_resource\_group\_name) | Monitor resource group name | `string` | n/a | yes | +| [prefix](#input\_prefix) | general | `string` | n/a | yes | +| [tags](#input\_tags) | n/a | `map(any)` | 
{
  "CreatedBy": "Terraform"
}
| no | + +## Outputs + +No outputs. + diff --git a/src/domains/gpdingestion-common/env/itn-dev/backend.ini b/src/domains/gpdingestion-common/env/itn-dev/backend.ini new file mode 100644 index 0000000000..f3ea2d530c --- /dev/null +++ b/src/domains/gpdingestion-common/env/itn-dev/backend.ini @@ -0,0 +1 @@ +subscription=DEV-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-common/env/itn-dev/backend.tfvars b/src/domains/gpdingestion-common/env/itn-dev/backend.tfvars new file mode 100644 index 0000000000..61e0c7b275 --- /dev/null +++ b/src/domains/gpdingestion-common/env/itn-dev/backend.tfvars @@ -0,0 +1,4 @@ +resource_group_name = "terraform-state-rg" +storage_account_name = "tfinfdevpagopa" +container_name = "terraform-state" +key = "gpdingestion-common-dev.terraform.tfstate" diff --git a/src/domains/gpdingestion-common/env/itn-dev/terraform.tfvars b/src/domains/gpdingestion-common/env/itn-dev/terraform.tfvars new file mode 100644 index 0000000000..f2714d8780 --- /dev/null +++ b/src/domains/gpdingestion-common/env/itn-dev/terraform.tfvars @@ -0,0 +1,59 @@ +prefix = "pagopa" +env_short = "d" +env = "dev" +domain = "gpdingestion" +location = "italynorth" +location_short = "itn" +instance = "dev" + +tags = { + CreatedBy = "Terraform" + Environment = "Dev" + Owner = "pagoPA" + Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-common" + CostCenter = "TS310 - PAGAMENTI & SERVIZI" +} + +### 🚩Features flags + +is_feature_enabled = { + eventhub = true +} + +### CIRDs + +cidr_gpdingestion_eventhub_italy = ["10.3.13.0/27"] + +### External resources + +monitor_italy_resource_group_name = "pagopa-d-itn-core-monitor-rg" +log_analytics_italy_workspace_name = "pagopa-d-itn-core-law" +log_analytics_italy_workspace_resource_group_name = "pagopa-d-itn-core-monitor-rg" + +monitor_resource_group_name = "pagopa-d-monitor-rg" +log_analytics_workspace_name = "pagopa-d-law" +log_analytics_workspace_resource_group_name = "pagopa-d-monitor-rg" + +### Aks + +ingress_load_balancer_ip = "10.3.100.250" + +external_domain = "pagopa.it" +dns_zone_internal_prefix = "internal.dev.platform" + +# +# EventHub +# +ehns_sku_name = "Standard" + +# to avoid https://docs.microsoft.com/it-it/azure/event-hubs/event-hubs-messaging-exceptions#error-code-50002 +ehns_auto_inflate_enabled = false +ehns_maximum_throughput_units = 5 +ehns_capacity = 1 +ehns_alerts_enabled = false +ehns_zone_redundant = false + +ehns_public_network_access = true +ehns_private_endpoint_is_present = false + + diff --git a/src/domains/gpdingestion-common/env/itn-prod/backend.ini b/src/domains/gpdingestion-common/env/itn-prod/backend.ini new file mode 100644 index 0000000000..432abea37c --- /dev/null +++ b/src/domains/gpdingestion-common/env/itn-prod/backend.ini @@ -0,0 +1 @@ +subscription=PROD-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-common/env/itn-prod/backend.tfvars b/src/domains/gpdingestion-common/env/itn-prod/backend.tfvars new file mode 100644 index 0000000000..6146f86e2c --- /dev/null +++ b/src/domains/gpdingestion-common/env/itn-prod/backend.tfvars @@ -0,0 +1,4 @@ +resource_group_name = "terraform-state-rg" +storage_account_name = "tfinfprodpagopa" +container_name = "terraform-state" +key = "gpdingestion-common-prod.terraform.tfstate" diff --git a/src/domains/gpdingestion-common/env/itn-prod/terraform.tfvars b/src/domains/gpdingestion-common/env/itn-prod/terraform.tfvars new file mode 100644 index 0000000000..47e0b7993f --- /dev/null +++ b/src/domains/gpdingestion-common/env/itn-prod/terraform.tfvars @@ -0,0 +1,58 @@ +prefix = "pagopa" +env_short = "p" +env = "prod" +domain = "payopt" +location = "italynorth" +location_short = "itn" +instance = "prod" + +tags = { + CreatedBy = "Terraform" + Environment = "Prod" + Owner = "pagoPA" + Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-common" + CostCenter = "TS310 - PAGAMENTI & SERVIZI" +} + +### 🚩Features flags + +is_feature_enabled = { + eventhub = true +} + +### CIRDs + +cidr_gpdingestion_eventhub_italy = ["10.3.13.0/27"] + +### External resources + +monitor_italy_resource_group_name = "pagopa-p-itn-core-monitor-rg" +log_analytics_italy_workspace_name = "pagopa-p-itn-core-law" +log_analytics_italy_workspace_resource_group_name = "pagopa-p-itn-core-monitor-rg" + +monitor_resource_group_name = "pagopa-p-monitor-rg" +log_analytics_workspace_name = "pagopa-p-law" +log_analytics_workspace_resource_group_name = "pagopa-p-monitor-rg" + +### Aks + +ingress_load_balancer_ip = "10.3.100.250" + +external_domain = "pagopa.it" +dns_zone_internal_prefix = "internal.platform" + +# +# EventHub +# +ehns_sku_name = "Standard" + +# to avoid https://docs.microsoft.com/it-it/azure/event-hubs/event-hubs-messaging-exceptions#error-code-50002 +ehns_auto_inflate_enabled = true +ehns_maximum_throughput_units = 5 +ehns_capacity = 5 +ehns_alerts_enabled = true +ehns_zone_redundant = true + +ehns_public_network_access = false +ehns_private_endpoint_is_present = true + diff --git a/src/domains/gpdingestion-common/env/itn-uat/backend.ini b/src/domains/gpdingestion-common/env/itn-uat/backend.ini new file mode 100644 index 0000000000..1759a0ca0d --- /dev/null +++ b/src/domains/gpdingestion-common/env/itn-uat/backend.ini @@ -0,0 +1 @@ +subscription=UAT-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-common/env/itn-uat/backend.tfvars b/src/domains/gpdingestion-common/env/itn-uat/backend.tfvars new file mode 100644 index 0000000000..3eca13f707 --- /dev/null +++ b/src/domains/gpdingestion-common/env/itn-uat/backend.tfvars @@ -0,0 +1,4 @@ +resource_group_name = "terraform-state-rg" +storage_account_name = "tfinfuatpagopa" +container_name = "terraform-state" +key = "gpdingestion-common-uat.terraform.tfstate" diff --git a/src/domains/gpdingestion-common/env/itn-uat/terraform.tfvars b/src/domains/gpdingestion-common/env/itn-uat/terraform.tfvars new file mode 100644 index 0000000000..019de1bf08 --- /dev/null +++ b/src/domains/gpdingestion-common/env/itn-uat/terraform.tfvars @@ -0,0 +1,58 @@ +prefix = "pagopa" +env_short = "u" +env = "uat" +domain = "gpdingestion" +location = "italynorth" +location_short = "itn" +instance = "uat" + +tags = { + CreatedBy = "Terraform" + Environment = "Uat" + Owner = "pagoPA" + Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-common" + CostCenter = "TS310 - PAGAMENTI & SERVIZI" +} + +### 🚩Features flags + +is_feature_enabled = { + eventhub = true +} + +### CIRDs + +cidr_gpdingestion_eventhub_italy = ["10.3.13.0/27"] + +### External resources + +monitor_italy_resource_group_name = "pagopa-u-itn-core-monitor-rg" +log_analytics_italy_workspace_name = "pagopa-u-itn-core-law" +log_analytics_italy_workspace_resource_group_name = "pagopa-u-itn-core-monitor-rg" + +monitor_resource_group_name = "pagopa-u-monitor-rg" +log_analytics_workspace_name = "pagopa-u-law" +log_analytics_workspace_resource_group_name = "pagopa-u-monitor-rg" + +### Aks + +ingress_load_balancer_ip = "10.3.100.250" + +external_domain = "pagopa.it" +dns_zone_internal_prefix = "internal.uat.platform" + +# +# EventHub +# +ehns_sku_name = "Standard" + +# to avoid https://docs.microsoft.com/it-it/azure/event-hubs/event-hubs-messaging-exceptions#error-code-50002 +ehns_auto_inflate_enabled = true +ehns_maximum_throughput_units = 5 +ehns_capacity = 1 +ehns_alerts_enabled = false +ehns_zone_redundant = false + +ehns_public_network_access = false +ehns_private_endpoint_is_present = true + diff --git a/src/domains/gpdingestion-common/terraform.sh b/src/domains/gpdingestion-common/terraform.sh new file mode 100755 index 0000000000..047a7512d0 --- /dev/null +++ b/src/domains/gpdingestion-common/terraform.sh @@ -0,0 +1,324 @@ +#!/bin/bash +############################################################ +# Terraform script for managing infrastructure on Azure +# Fingerprint: d2hhdHlvdXdhbnQ/Cg== +############################################################ +# Global variables +# Version format x.y accepted +vers="1.11" +script_name=$(basename "$0") +git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/azure/${script_name}" +tmp_file="${script_name}.new" +# Check if the third parameter exists and is a file +if [ -n "$3" ] && [ -f "$3" ]; then + FILE_ACTION=true +else + FILE_ACTION=false +fi + +# Define functions +function clean_environment() { + rm -rf .terraform + rm tfplan 2>/dev/null + echo "cleaned!" +} + +function download_tool() { + #default value + cpu_type="intel" + os_type=$(uname) + + # only on MacOS + if [ "$os_type" == "Darwin" ]; then + cpu_brand=$(sysctl -n machdep.cpu.brand_string) + if grep -q -i "intel" <<< "$cpu_brand"; then + cpu_type="intel" + else + cpu_type="arm" + fi + fi + + echo $cpu_type + tool=$1 + git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/golang/${tool}_${cpu_type}" + if ! command -v $tool &> /dev/null; then + if ! curl -sL "$git_repo" -o "$tool"; then + echo "Error downloading ${tool}" + return 1 + else + chmod +x $tool + echo "${tool} downloaded! Please note this tool WON'T be copied in your **/bin folder for safety reasons. +You need to do it yourself!" + read -p "Press enter to continue" + + + fi + fi +} + +function extract_resources() { + TF_FILE=$1 + ENV=$2 + TARGETS="" + + # Check if the file exists + if [ ! -f "$TF_FILE" ]; then + echo "File $TF_FILE does not exist." + exit 1 + fi + + # Check if the directory exists + if [ ! -d "./env/$ENV" ]; then + echo "Directory ./env/$ENV does not exist." + exit 1 + fi + + TMP_FILE=$(mktemp) + grep -E '^resource|^module' $TF_FILE > $TMP_FILE + + while read -r line ; do + TYPE=$(echo $line | cut -d '"' -f 1 | tr -d ' ') + if [ "$TYPE" == "module" ]; then + NAME=$(echo $line | cut -d '"' -f 2) + TARGETS+=" -target=\"$TYPE.$NAME\"" + else + NAME1=$(echo $line | cut -d '"' -f 2) + NAME2=$(echo $line | cut -d '"' -f 4) + TARGETS+=" -target=\"$NAME1.$NAME2\"" + fi + done < $TMP_FILE + + rm $TMP_FILE + + echo "./terraform.sh $action $ENV $TARGETS" +} + +function help_usage() { + echo "terraform.sh Version ${vers}" + echo + echo "Usage: ./script.sh [ACTION] [ENV] [OTHER OPTIONS]" + echo "es. ACTION: init, apply, plan, etc." + echo "es. ENV: dev, uat, prod, etc." + echo + echo "Available actions:" + echo " clean Remove .terraform* folders and tfplan files" + echo " help This help" + echo " list List every environment available" + echo " update Update this script if possible" + echo " summ Generate summary of Terraform plan" + echo " tflist Generate an improved output of terraform state list" + echo " tlock Generate or update the dependency lock file" + echo " * any terraform option" +} + +function init_terraform() { + if [ -n "$env" ]; then + terraform init -reconfigure -backend-config="./env/$env/backend.tfvars" + else + echo "ERROR: no env configured!" + exit 1 + fi +} + +function list_env() { + # Check if env directory exists + if [ ! -d "./env" ]; then + echo "No environment directory found" + exit 1 + fi + + # List subdirectories under env directory + env_list=$(ls -d ./env/*/ 2>/dev/null) + + # Check if there are any subdirectories + if [ -z "$env_list" ]; then + echo "No environments found" + exit 1 + fi + + # Print the list of environments + echo "Available environments:" + for env in $env_list; do + env_name=$(echo "$env" | sed 's#./env/##;s#/##') + echo "- $env_name" + done +} + +function other_actions() { + if [ -n "$env" ] && [ -n "$action" ]; then + terraform "$action" -var-file="./env/$env/terraform.tfvars" -compact-warnings $other + else + echo "ERROR: no env or action configured!" + exit 1 + fi +} + +function state_output_taint_actions() { + if [ "$action" == "tflist" ]; then + # If 'tflist' is not installed globally and there is no 'tflist' file in the current directory, + # attempt to download the 'tflist' tool + if ! command -v tflist &> /dev/null && [ ! -f "tflist" ]; then + download_tool "tflist" + if [ $? -ne 0 ]; then + echo "Error: Failed to download tflist!!" + exit 1 + else + echo "tflist downloaded!" + fi + fi + if command -v tflist &> /dev/null; then + terraform state list | tflist + else + terraform state list | ./tflist + fi + else + terraform $action $other + fi +} + + +function parse_tfplan_option() { + # Create an array to contain arguments that do not start with '-tfplan=' + local other_args=() + + # Loop over all arguments + for arg in "$@"; do + # If the argument starts with '-tfplan=', extract the file name + if [[ "$arg" =~ ^-tfplan= ]]; then + echo "${arg#*=}" + else + # If the argument does not start with '-tfplan=', add it to the other_args array + other_args+=("$arg") + fi + done + + # Print all arguments in other_args separated by spaces + echo "${other_args[@]}" +} + +function tfsummary() { + local plan_file + plan_file=$(parse_tfplan_option "$@") + if [ -z "$plan_file" ]; then + plan_file="tfplan" + fi + action="plan" + other="-out=${plan_file}" + other_actions + if [ -n "$(command -v tf-summarize)" ]; then + tf-summarize -tree "${plan_file}" + else + echo "tf-summarize is not installed" + fi + if [ "$plan_file" == "tfplan" ]; then + rm $plan_file + fi +} + +function update_script() { + # Check if the repository was cloned successfully + if ! curl -sL "$git_repo" -o "$tmp_file"; then + echo "Error cloning the repository" + rm "$tmp_file" 2>/dev/null + return 1 + fi + + # Check if a newer version exists + remote_vers=$(sed -n '8s/vers="\(.*\)"/\1/p' "$tmp_file") + if [ "$(printf '%s\n' "$vers" "$remote_vers" | sort -V | tail -n 1)" == "$vers" ]; then + echo "The local script version is equal to or newer than the remote version." + rm "$tmp_file" 2>/dev/null + return 0 + fi + + # Check the fingerprint + local_fingerprint=$(sed -n '4p' "$0") + remote_fingerprint=$(sed -n '4p' "$tmp_file") + + if [ "$local_fingerprint" != "$remote_fingerprint" ]; then + echo "The local and remote file fingerprints do not match." + rm "$tmp_file" 2>/dev/null + return 0 + fi + + # Show the current and available versions to the user + echo "Current script version: $vers" + echo "Available script version: $remote_vers" + + # Ask the user if they want to update the script + read -rp "Do you want to update the script to version $remote_vers? (y/n): " answer + + if [ "$answer" == "y" ] || [ "$answer" == "Y" ]; then + # Replace the local script with the updated version + cp "$tmp_file" "$script_name" + chmod +x "$script_name" + rm "$tmp_file" 2>/dev/null + + echo "Script successfully updated to version $remote_vers" + else + echo "Update canceled by the user" + fi + + rm "$tmp_file" 2>/dev/null +} + +# Check arguments number +if [ "$#" -lt 1 ]; then + help_usage + exit 0 +fi + +# Parse arguments +action=$1 +env=$2 +filetf=$3 +shift 2 +other=$@ + +if [ -n "$env" ]; then + # shellcheck source=/dev/null + source "./env/$env/backend.ini" + if [ -z "$(command -v az)" ]; then + echo "az not found, cannot proceed" + exit 1 + fi + az account set -s "${subscription}" +fi + +# Call appropriate function based on action +case $action in + clean) + clean_environment + ;; + ?|help|-h) + help_usage + ;; + init) + init_terraform "$other" + ;; + list) + list_env + ;; + output|state|taint|tflist) + init_terraform + state_output_taint_actions $other + ;; + summ) + init_terraform + tfsummary "$other" + ;; + tlock) + terraform providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=darwin_arm64 -platform=linux_amd64 + ;; + update) + update_script + ;; + *) + if [ "$FILE_ACTION" = true ]; then + extract_resources "$filetf" "$env" + else + init_terraform + other_actions "$other" + fi + ;; +esac diff --git a/src/domains/gpdingestion-secrets/.terraform.lock.hcl b/src/domains/gpdingestion-secrets/.terraform.lock.hcl new file mode 100644 index 0000000000..a389468af5 --- /dev/null +++ b/src/domains/gpdingestion-secrets/.terraform.lock.hcl @@ -0,0 +1,107 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "2.47.0" + constraints = "<= 2.47.0" + hashes = [ + "h1:g8+gBFM4QVOEQFqAEs5pR6iXpbGvgPvcEi1evHwziyw=", + "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", + "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", + "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", + "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", + "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", + "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", + "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", + "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", + "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", + "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", + "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.106.0" + constraints = "~> 3.30, <= 3.106.0" + hashes = [ + "h1:6t9Nz9tYAR9BfHZ8yc56m+GKRl0nriwjQ5DyA0/TnCs=", + "h1:Mxe1/I27IZK3BP6cm84Gt0+7PXd2EDaDUMxuljm/rUA=", + "zh:07980d6fdc40c0adb670c8413a5c667917d6dbb51fcedc467c35d64c2f3a1f47", + "zh:2e6e8491b1f089644b0d23f8da83398f1e10cf5a62b16efcef2b5454fe923038", + "zh:450dbd72821c5619cc3bcdc20fdd0e29515147e44b733f9c79d3a75851810055", + "zh:5e234c0a2f3c9677ea72b2a6e6ca90defb99fab29ae565f5d1f70728ba4ba78f", + "zh:83fd042ece6977429d79affd03d6ce963d2f122604dbf15a1abf203d7a7bbc8a", + "zh:93027e1f66b3bf83398d572d4e6f6e7777330c78c54da3226dadd50fd868ada9", + "zh:ae3d1dd66140c303df97d93c47a60f16735ce17cf156f45475dcee4a7360af5b", + "zh:daf9d2eb89e785458a76b88bf2ef0696c472094c77cc9cff3b3ea4b885c5a482", + "zh:dd46370141651e6549da6d85e25c7a6770c47581bbaaa27eda2886d41d849747", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f77405c0d8f6e0d93d9da83256b3b02c164bad4c791ed9604310ff02ae086ad1", + "zh:ffa769147bda833aef8802e3a391bd175ec749862764d61cbdaa8200d5b8f893", + ] +} + +provider "registry.terraform.io/hashicorp/external" { + version = "2.2.3" + constraints = "<= 2.2.3" + hashes = [ + "h1:648ZjJR81c2W1OLtYmUQa9/1rGr3vvZSuX9dR1ucGWY=", + "h1:D2RKjqoU26isFINpmeKG9NS0LvkPmrQkNXeYO2TdgyA=", + "zh:184ecd339d764de845db0e5b8a9c87893dcd0c9d822167f73658f89d80ec31c9", + "zh:2661eaca31d17d6bbb18a8f673bbfe3fe1b9b7326e60d0ceb302017003274e3c", + "zh:2c0a180f6d1fc2ba6e03f7dfc5f73b617e45408681f75bca75aa82f3796df0e4", + "zh:4b92ae44c6baef4c4952c47be00541055cb5280dd3bc8031dba5a1b2ee982387", + "zh:5641694d5daf3893d7ea90be03b6fa575211a08814ffe70998d5adb8b59cdc0a", + "zh:5bd55a2be8a1c20d732ac9c604b839e1cadc8c49006315dffa4d709b6874df32", + "zh:6e0ef5d11e1597202424b7d69b9da7b881494c9b13a3d4026fc47012dc651c79", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9e19f89fa25004d3b926a8d15ea630b4bde62f1fa4ed5e11a3d27aabddb77353", + "zh:b763efdd69fd097616b4a4c89cf333b4cee9699ac6432d73d2756f8335d1213f", + "zh:e3b561efdee510b2b445f76a52a902c52bee8e13095e7f4bed7c80f10f8d294a", + "zh:fe660bb8781ee043a093b9a20e53069974475dcaa5791a1f45fd03c61a26478a", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.16.1" + constraints = "<= 2.16.1" + hashes = [ + "h1:PO4Ye/+lu5hCaUEOtwNOldQYoA0dqL1bcBICIpdlcd8=", + "h1:kO/d+ZMZYM2tNMMFHZqBmVR0MeemoGnI2G2NSN92CrU=", + "zh:06224975f5910d41e73b35a4d5079861da2c24f9353e3ebb015fbb3b3b996b1c", + "zh:2bc400a8d9fe7755cca27c2551564a9e2609cfadc77f526ef855114ee02d446f", + "zh:3a479014187af1d0aec3a1d3d9c09551b801956fe6dd29af1186dec86712731b", + "zh:73fb0a69f1abdb02858b6589f7fab6d989a0f422f7ad95ed662aaa84872d3473", + "zh:a33852cd382cbc8e06d3f6c018b468ad809d24d912d64722e037aed1f9bf39db", + "zh:b533ff2214dca90296b1d22eace7eaa7e3efe5a7ae9da66a112094abc932db4f", + "zh:ddf74d8bb1aeb01dc2c36ef40e2b283d32b2a96db73f6daaf179fa2f10949c80", + "zh:e720f3a15d34e795fa9ff90bc755e838ebb4aef894aa2a423fb16dfa6d6b0667", + "zh:e789ae70a658800cb0a19ef7e4e9b26b5a38a92b43d1f41d64fc8bb46539cefb", + "zh:e8aed7dc0bd8f843d607dee5f72640dbef6835a8b1c6ea12cea5b4ec53e463f7", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fb3ac4f43c8b0dfc0b0103dd0f062ea72b3a34518d4c8808e3a44c9a3dd5f024", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.1" + constraints = "~> 3.2, <= 3.2.1" + hashes = [ + "h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=", + "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", + "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", + "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", + "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", + "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", + "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", + "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", + "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", + "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", + "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", + "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", + ] +} diff --git a/src/domains/gpdingestion-secrets/00_azuread.tf b/src/domains/gpdingestion-secrets/00_azuread.tf new file mode 100644 index 0000000000..14a0893a9f --- /dev/null +++ b/src/domains/gpdingestion-secrets/00_azuread.tf @@ -0,0 +1,16 @@ +# Azure AD +data "azuread_group" "adgroup_admin" { + display_name = "${local.product}-adgroup-admin" +} + +data "azuread_group" "adgroup_developers" { + display_name = "${local.product}-adgroup-developers" +} + +data "azuread_group" "adgroup_externals" { + display_name = "${local.product}-adgroup-externals" +} + +data "azuread_group" "adgroup_security" { + display_name = "${local.product}-adgroup-security" +} \ No newline at end of file diff --git a/src/domains/gpdingestion-secrets/01_keyvault.tf b/src/domains/gpdingestion-secrets/01_keyvault.tf new file mode 100644 index 0000000000..e6a4a6322b --- /dev/null +++ b/src/domains/gpdingestion-secrets/01_keyvault.tf @@ -0,0 +1,101 @@ +resource "azurerm_resource_group" "sec_rg" { + name = "${local.product}-${var.location_short}-${var.domain}-sec-rg" + location = var.location + + tags = var.tags +} + +module "key_vault" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//key_vault?ref=v8.22.0" + + name = "${local.product}-${var.location_short}-${var.domain}-kv" + location = azurerm_resource_group.sec_rg.location + resource_group_name = azurerm_resource_group.sec_rg.name + tenant_id = data.azurerm_client_config.current.tenant_id + soft_delete_retention_days = 90 + + tags = var.tags +} + +## ad group policy ## +resource "azurerm_key_vault_access_policy" "ad_group_policy" { + key_vault_id = module.key_vault.id + + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azuread_group.adgroup_admin.object_id + + key_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Encrypt", "Decrypt", "Backup", "Purge", "Recover", "Restore", "Sign", "UnwrapKey", "Update", "Verify", "WrapKey", "Release", "Rotate", "GetRotationPolicy", "SetRotationPolicy"] + secret_permissions = ["Get", "List", "Set", "Delete", "Backup", "Purge", "Recover", "Restore"] + storage_permissions = [] + certificate_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Restore", "Purge", "Recover", ] +} + +## ad group policy ## +resource "azurerm_key_vault_access_policy" "adgroup_developers_policy" { + count = var.env_short != "p" ? 1 : 0 + + key_vault_id = module.key_vault.id + + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azuread_group.adgroup_developers.object_id + + key_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Encrypt", "Decrypt", "Recover", "Rotate", "GetRotationPolicy"] + secret_permissions = ["Get", "List", "Set", "Delete", "Recover", ] + storage_permissions = [] + certificate_permissions = [ + "Get", "List", "Update", "Create", "Import", + "Delete", "Restore", "Purge", "Recover" + ] +} + +## ad group policy ## +resource "azurerm_key_vault_access_policy" "adgroup_externals_policy" { + count = var.env_short != "p" ? 1 : 0 + + key_vault_id = module.key_vault.id + + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azuread_group.adgroup_externals.object_id + + key_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Encrypt", "Decrypt", "Recover", "Rotate", "GetRotationPolicy"] + secret_permissions = ["Get", "List", "Set", "Delete", "Recover", ] + storage_permissions = [] + certificate_permissions = [ + "Get", "List", "Update", "Create", "Import", + "Delete", "Restore", "Purge", "Recover" + ] +} + +## ad group policy ## +data "azuread_service_principal" "iac_principal" { + count = var.enable_iac_pipeline ? 1 : 0 + display_name = "pagopaspa-pagoPA-iac-${data.azurerm_subscription.current.subscription_id}" +} + +resource "azurerm_key_vault_access_policy" "azdevops_iac_policy" { + count = var.enable_iac_pipeline ? 1 : 0 + key_vault_id = module.key_vault.id + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azuread_service_principal.iac_principal[0].object_id + + secret_permissions = ["Get", "List", "Set", ] + certificate_permissions = ["SetIssuers", "DeleteIssuers", "Purge", "List", "Get"] + key_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Encrypt", "Decrypt"] + + storage_permissions = [] +} + +################ +## Secrets ## +################ + +# create json letsencrypt inside kv +# requierd: Docker +module "letsencrypt_gpdingestion" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git///letsencrypt_credential?ref=v8.44.0" + + prefix = var.prefix + env = var.env_short + key_vault_name = module.key_vault.name + subscription_name = local.subscription_name +} diff --git a/src/domains/gpdingestion-secrets/02_azdo.tf b/src/domains/gpdingestion-secrets/02_azdo.tf new file mode 100644 index 0000000000..5683ffec89 --- /dev/null +++ b/src/domains/gpdingestion-secrets/02_azdo.tf @@ -0,0 +1,23 @@ +# +# Policy +# + +data "azurerm_user_assigned_identity" "iac_federated_azdo" { + for_each = local.azdo_iac_managed_identities + name = each.key + resource_group_name = local.azdo_managed_identity_rg_name +} + +resource "azurerm_key_vault_access_policy" "azdevops_iac_managed_identities" { + for_each = local.azdo_iac_managed_identities + + key_vault_id = module.key_vault.id + tenant_id = data.azurerm_client_config.current.tenant_id + object_id = data.azurerm_user_assigned_identity.iac_federated_azdo[each.key].principal_id + + secret_permissions = ["Get", "List", "Set", ] + + certificate_permissions = ["SetIssuers", "DeleteIssuers", "Purge", "List", "Get"] + + storage_permissions = [] +} diff --git a/src/domains/gpdingestion-secrets/02_init_sops.tf b/src/domains/gpdingestion-secrets/02_init_sops.tf new file mode 100644 index 0000000000..e93d0651a0 --- /dev/null +++ b/src/domains/gpdingestion-secrets/02_init_sops.tf @@ -0,0 +1,21 @@ +moved { + from = azurerm_key_vault_key.generated + to = azurerm_key_vault_key.sops_key +} + +resource "azurerm_key_vault_key" "sops_key" { + name = "${local.product}-${var.domain}-sops-key" + key_vault_id = module.key_vault.id + key_type = "RSA" + key_size = 2048 + + key_opts = [ + "decrypt", + "encrypt", + ] + + depends_on = [ + azurerm_key_vault_access_policy.adgroup_developers_policy, + azurerm_key_vault_access_policy.ad_group_policy, + ] +} diff --git a/src/domains/gpdingestion-secrets/03_sops_secrets.tf b/src/domains/gpdingestion-secrets/03_sops_secrets.tf new file mode 100644 index 0000000000..68c06265a2 --- /dev/null +++ b/src/domains/gpdingestion-secrets/03_sops_secrets.tf @@ -0,0 +1,54 @@ +moved { + from = data.external.external2 + to = data.external.terrasops +} + +data "external" "terrasops" { + program = [ + "bash", "terrasops.sh" + ] + query = { + env = "${var.location_short}-${var.env}" + } + +} + +locals { + all_enc_secrets_value = can(data.external.terrasops.result) ? flatten([ + for k, v in data.external.terrasops.result : { + valore = v + chiave = k + } + ]) : [] + + config_secret_data = jsondecode(file(var.input_file)) + all_config_secrets_value = flatten([ + for kc, vc in local.config_secret_data : { + valore = vc + chiave = kc + } + ]) + + all_secrets_value = concat(local.all_config_secrets_value, local.all_enc_secrets_value) +} + +## SOPS secrets + +## Upload all encrypted secrets +resource "azurerm_key_vault_secret" "secret" { + for_each = { for i, v in local.all_secrets_value : local.all_secrets_value[i].chiave => i } + + key_vault_id = module.key_vault.id + name = local.all_secrets_value[each.value].chiave + value = local.all_secrets_value[each.value].valore + + depends_on = [ + module.key_vault, + azurerm_key_vault_key.sops_key, + data.external.terrasops, + azurerm_key_vault_access_policy.adgroup_developers_policy, + azurerm_key_vault_access_policy.ad_group_policy, + ] +} + +# ⚠️ The secrets from resources are set in gpdingestion-app to avoid circular dependency diff --git a/src/domains/gpdingestion-secrets/99_locals.tf b/src/domains/gpdingestion-secrets/99_locals.tf new file mode 100644 index 0000000000..084cb86f0c --- /dev/null +++ b/src/domains/gpdingestion-secrets/99_locals.tf @@ -0,0 +1,11 @@ +locals { + project = "${var.prefix}-${var.env_short}-${var.location_short}-${var.domain}" + product = "${var.prefix}-${var.env_short}" + + + subscription_name = "${var.env}-${var.prefix}" + + azdo_managed_identity_rg_name = "pagopa-${var.env_short}-identity-rg" + azdo_iac_managed_identities = toset(["azdo-${var.env}-pagopa-iac-deploy", "azdo-${var.env}-pagopa-iac-plan"]) + +} diff --git a/src/domains/gpdingestion-secrets/99_main.tf b/src/domains/gpdingestion-secrets/99_main.tf new file mode 100644 index 0000000000..93ec8f61ca --- /dev/null +++ b/src/domains/gpdingestion-secrets/99_main.tf @@ -0,0 +1,43 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "<= 3.106.0" + } + azuread = { + source = "hashicorp/azuread" + version = "<= 2.47.0" + } + null = { + source = "hashicorp/null" + version = "<= 3.2.1" + } + external = { + source = "hashicorp/external" + version = "<= 2.2.3" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "<= 2.16.1" + } + } + + backend "azurerm" {} +} + +provider "azurerm" { + features { + key_vault { + purge_soft_delete_on_destroy = false + } + } +} + +provider "kubernetes" { + config_path = "~/.kube/config-${var.prefix}-${var.env_short}-${var.location_short}-${var.env}-aks" + config_context = "${var.prefix}-${var.env_short}-${var.location_short}-${var.env}-aks" +} + +data "azurerm_subscription" "current" {} + +data "azurerm_client_config" "current" {} diff --git a/src/domains/gpdingestion-secrets/99_variables.tf b/src/domains/gpdingestion-secrets/99_variables.tf new file mode 100644 index 0000000000..3a7cff7fcf --- /dev/null +++ b/src/domains/gpdingestion-secrets/99_variables.tf @@ -0,0 +1,101 @@ +# general + +variable "prefix" { + type = string + validation { + condition = ( + length(var.prefix) <= 6 + ) + error_message = "Max length is 6 chars." + } +} + +variable "env" { + type = string +} + +variable "env_short" { + type = string + validation { + condition = ( + length(var.env_short) == 1 + ) + error_message = "Length must be 1 chars." + } +} + +variable "domain" { + type = string + validation { + condition = ( + length(var.domain) <= 12 + ) + error_message = "Max length is 12 chars." + } +} + +variable "location" { + type = string + description = "One of westeurope, northeurope" +} + +variable "location_short" { + type = string + validation { + condition = ( + length(var.location_short) == 3 + ) + error_message = "Length must be 3 chars." + } + description = "One of weu, itn" +} + +variable "instance" { + type = string + description = "One of beta, prod01, prod02" +} + +variable "tags" { + type = map(any) + default = { + CreatedBy = "Terraform" + } +} + +### + +variable "input_file" { + type = string + description = "secret json file" +} + +variable "enable_iac_pipeline" { + type = bool + description = "If true create the key vault policy to allow used by azure devops iac pipelines." + default = false +} + + +variable "kv-key-permissions-read" { + type = list(string) + description = "List of read key permissions" + default = ["Get", "List"] +} + +variable "kv-secret-permissions-read" { + type = list(string) + description = "List of read secret permissions" + default = ["Get", "List"] +} + +variable "kv-certificate-permissions-read" { + type = list(string) + description = "List of read certificate permissions" + default = ["Get", "GetIssuers", "List", "ListIssuers"] +} + +variable "kv-storage-permissions-read" { + type = list(string) + description = "List of read storage permissions" + default = ["Get", "GetSAS", "List", "ListSAS"] +} diff --git a/src/domains/gpdingestion-secrets/README.md b/src/domains/gpdingestion-secrets/README.md new file mode 100644 index 0000000000..ccd6f2a60c --- /dev/null +++ b/src/domains/gpdingestion-secrets/README.md @@ -0,0 +1,65 @@ +# paymentoptions-secrets + + +## Requirements + +| Name | Version | +|------|---------| +| [azuread](#requirement\_azuread) | <= 2.47.0 | +| [azurerm](#requirement\_azurerm) | <= 3.106.0 | +| [external](#requirement\_external) | <= 2.2.3 | +| [kubernetes](#requirement\_kubernetes) | <= 2.16.1 | +| [null](#requirement\_null) | <= 3.2.1 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [key\_vault](#module\_key\_vault) | git::https://github.com/pagopa/terraform-azurerm-v3.git//key_vault | v8.22.0 | +| [letsencrypt\_paymentoptions](#module\_letsencrypt\_paymentoptions) | git::https://github.com/pagopa/terraform-azurerm-v3.git///letsencrypt_credential | v8.44.0 | + +## Resources + +| Name | Type | +|------|------| +| [azurerm_key_vault_access_policy.ad_group_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | +| [azurerm_key_vault_access_policy.adgroup_developers_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | +| [azurerm_key_vault_access_policy.adgroup_externals_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | +| [azurerm_key_vault_access_policy.azdevops_iac_managed_identities](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | +| [azurerm_key_vault_access_policy.azdevops_iac_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | +| [azurerm_key_vault_key.sops_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_key) | resource | +| [azurerm_key_vault_secret.secret](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_resource_group.sec_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | +| [azuread_group.adgroup_admin](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | +| [azuread_group.adgroup_developers](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | +| [azuread_group.adgroup_externals](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | +| [azuread_group.adgroup_security](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | +| [azuread_service_principal.iac_principal](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/service_principal) | data source | +| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | +| [azurerm_user_assigned_identity.iac_federated_azdo](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/user_assigned_identity) | data source | +| [external_external.terrasops](https://registry.terraform.io/providers/hashicorp/external/latest/docs/data-sources/external) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [domain](#input\_domain) | n/a | `string` | n/a | yes | +| [enable\_iac\_pipeline](#input\_enable\_iac\_pipeline) | If true create the key vault policy to allow used by azure devops iac pipelines. | `bool` | `false` | no | +| [env](#input\_env) | n/a | `string` | n/a | yes | +| [env\_short](#input\_env\_short) | n/a | `string` | n/a | yes | +| [input\_file](#input\_input\_file) | secret json file | `string` | n/a | yes | +| [instance](#input\_instance) | One of beta, prod01, prod02 | `string` | n/a | yes | +| [kv-certificate-permissions-read](#input\_kv-certificate-permissions-read) | List of read certificate permissions | `list(string)` | 
[
  "Get",
  "GetIssuers",
  "List",
  "ListIssuers"
]
| no | +| [kv-key-permissions-read](#input\_kv-key-permissions-read) | List of read key permissions | `list(string)` | 
[
  "Get",
  "List"
]
| no | +| [kv-secret-permissions-read](#input\_kv-secret-permissions-read) | List of read secret permissions | `list(string)` | 
[
  "Get",
  "List"
]
| no | +| [kv-storage-permissions-read](#input\_kv-storage-permissions-read) | List of read storage permissions | `list(string)` | 
[
  "Get",
  "GetSAS",
  "List",
  "ListSAS"
]
| no | +| [location](#input\_location) | One of westeurope, northeurope | `string` | n/a | yes | +| [location\_short](#input\_location\_short) | One of weu, itn | `string` | n/a | yes | +| [prefix](#input\_prefix) | n/a | `string` | n/a | yes | +| [tags](#input\_tags) | n/a | `map(any)` | 
{
  "CreatedBy": "Terraform"
}
| no | + +## Outputs + +No outputs. + diff --git a/src/domains/gpdingestion-secrets/env/itn-dev/backend.ini b/src/domains/gpdingestion-secrets/env/itn-dev/backend.ini new file mode 100644 index 0000000000..f3ea2d530c --- /dev/null +++ b/src/domains/gpdingestion-secrets/env/itn-dev/backend.ini @@ -0,0 +1 @@ +subscription=DEV-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-secrets/env/itn-dev/backend.tfvars b/src/domains/gpdingestion-secrets/env/itn-dev/backend.tfvars new file mode 100644 index 0000000000..dfd890cc01 --- /dev/null +++ b/src/domains/gpdingestion-secrets/env/itn-dev/backend.tfvars @@ -0,0 +1,4 @@ +resource_group_name = "terraform-state-rg" +storage_account_name = "tfinfdevpagopa" +container_name = "terraform-state" +key = "gpdingestion-secret-dev.terraform.tfstate" diff --git a/src/domains/gpdingestion-secrets/env/itn-dev/terraform.tfvars b/src/domains/gpdingestion-secrets/env/itn-dev/terraform.tfvars new file mode 100644 index 0000000000..3fc2637c4c --- /dev/null +++ b/src/domains/gpdingestion-secrets/env/itn-dev/terraform.tfvars @@ -0,0 +1,30 @@ +prefix = "pagopa" +env_short = "d" +env = "dev" +domain = "gpdingestion" +location = "italynorth" +location_short = "itn" +instance = "dev" + +tags = { + CreatedBy = "Terraform" + Environment = "Dev" + Owner = "pagoPA" + Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-secrets" + CostCenter = "TS310 - PAGAMENTI & SERVIZI" +} + +### External resources + +monitor_italy_resource_group_name = "pagopa-d-itn-core-monitor-rg" +log_analytics_italy_workspace_name = "pagopa-d-itn-core-law" +log_analytics_italy_workspace_resource_group_name = "pagopa-d-itn-core-monitor-rg" + +input_file = "./secret/itn-dev/configs.json" + +enable_iac_pipeline = true + + + + + diff --git a/src/domains/gpdingestion-secrets/env/itn-prod/backend.ini b/src/domains/gpdingestion-secrets/env/itn-prod/backend.ini new file mode 100644 index 0000000000..6318425346 --- /dev/null +++ b/src/domains/gpdingestion-secrets/env/itn-prod/backend.ini @@ -0,0 +1 @@ +subscription=PROD-pagoPA diff --git a/src/domains/gpdingestion-secrets/env/itn-prod/backend.tfvars b/src/domains/gpdingestion-secrets/env/itn-prod/backend.tfvars new file mode 100644 index 0000000000..9277ee7b79 --- /dev/null +++ b/src/domains/gpdingestion-secrets/env/itn-prod/backend.tfvars @@ -0,0 +1,4 @@ +resource_group_name = "terraform-state-rg" +storage_account_name = "tfinfprodpagopa" +container_name = "terraform-state" +key = "gpdingestion-secret-prod.terraform.tfstate" diff --git a/src/domains/gpdingestion-secrets/env/itn-prod/terraform.tfvars b/src/domains/gpdingestion-secrets/env/itn-prod/terraform.tfvars new file mode 100644 index 0000000000..8f9451ab76 --- /dev/null +++ b/src/domains/gpdingestion-secrets/env/itn-prod/terraform.tfvars @@ -0,0 +1,30 @@ +prefix = "pagopa" +env_short = "p" +env = "prod" +domain = "gpdingestion" +location = "gpdingestion" +location_short = "itn" +instance = "prod" + +tags = { + CreatedBy = "Terraform" + Environment = "Prod" + Owner = "pagoPA" + Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-secrets" + CostCenter = "TS310 - PAGAMENTI & SERVIZI" +} + +### External resources + +monitor_italy_resource_group_name = "pagopa-d-itn-core-monitor-rg" +log_analytics_italy_workspace_name = "pagopa-d-itn-core-law" +log_analytics_italy_workspace_resource_group_name = "pagopa-d-itn-core-monitor-rg" + +input_file = "./secret/itn-prod/configs.json" + +enable_iac_pipeline = true + + + + + diff --git a/src/domains/gpdingestion-secrets/env/itn-uat/backend.ini b/src/domains/gpdingestion-secrets/env/itn-uat/backend.ini new file mode 100644 index 0000000000..1a014151dc --- /dev/null +++ b/src/domains/gpdingestion-secrets/env/itn-uat/backend.ini @@ -0,0 +1 @@ +subscription=UAT-pagoPA diff --git a/src/domains/gpdingestion-secrets/env/itn-uat/backend.tfvars b/src/domains/gpdingestion-secrets/env/itn-uat/backend.tfvars new file mode 100644 index 0000000000..e2a7d61cf4 --- /dev/null +++ b/src/domains/gpdingestion-secrets/env/itn-uat/backend.tfvars @@ -0,0 +1,4 @@ +resource_group_name = "terraform-state-rg" +storage_account_name = "tfinfuatpagopa" +container_name = "terraform-state" +key = "gpdingestion-secret-uat.terraform.tfstate" diff --git a/src/domains/gpdingestion-secrets/env/itn-uat/terraform.tfvars b/src/domains/gpdingestion-secrets/env/itn-uat/terraform.tfvars new file mode 100644 index 0000000000..7cb0ea8146 --- /dev/null +++ b/src/domains/gpdingestion-secrets/env/itn-uat/terraform.tfvars @@ -0,0 +1,27 @@ +prefix = "pagopa" +env_short = "u" +env = "uat" +domain = "gpdingestion" +location = "italynorth" +location_short = "itn" +instance = "uat" + +tags = { + CreatedBy = "Terraform" + Environment = "Uat" + Owner = "pagoPA" + Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-secrets" + CostCenter = "TS310 - PAGAMENTI & SERVIZI" +} + +### External resources + +monitor_italy_resource_group_name = "pagopa-d-itn-core-monitor-rg" +log_analytics_italy_workspace_name = "pagopa-d-itn-core-law" +log_analytics_italy_workspace_resource_group_name = "pagopa-d-itn-core-monitor-rg" + +input_file = "./secret/itn-uat/configs.json" + +enable_iac_pipeline = true + +force = "v1" diff --git a/src/domains/gpdingestion-secrets/secret/itn-dev/configs.json b/src/domains/gpdingestion-secrets/secret/itn-dev/configs.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/src/domains/gpdingestion-secrets/secret/itn-dev/configs.json @@ -0,0 +1 @@ +{} diff --git a/src/domains/gpdingestion-secrets/secret/itn-dev/secret.ini b/src/domains/gpdingestion-secrets/secret/itn-dev/secret.ini new file mode 100644 index 0000000000..5152851be5 --- /dev/null +++ b/src/domains/gpdingestion-secrets/secret/itn-dev/secret.ini @@ -0,0 +1,3 @@ +file_crypted="noedit_secret_enc.json" +kv_name="pagopa-d-itn-gpdingestion-kv" +kv_sops_key_name="pagopa-d-gpdingestion-sops-key" diff --git a/src/domains/gpdingestion-secrets/secret/itn-prod/configs.json b/src/domains/gpdingestion-secrets/secret/itn-prod/configs.json new file mode 100644 index 0000000000..2c63c08510 --- /dev/null +++ b/src/domains/gpdingestion-secrets/secret/itn-prod/configs.json @@ -0,0 +1,2 @@ +{ +} diff --git a/src/domains/gpdingestion-secrets/secret/itn-prod/secret.ini b/src/domains/gpdingestion-secrets/secret/itn-prod/secret.ini new file mode 100644 index 0000000000..f4b304862a --- /dev/null +++ b/src/domains/gpdingestion-secrets/secret/itn-prod/secret.ini @@ -0,0 +1,3 @@ +file_crypted="noedit_secret_enc.json" +kv_name="pagopa-p-itn-gpdingestion-kv" +kv_sops_key_name="pagopa-p-gpdingestion-sops-key" diff --git a/src/domains/gpdingestion-secrets/secret/itn-uat/configs.json b/src/domains/gpdingestion-secrets/secret/itn-uat/configs.json new file mode 100644 index 0000000000..2c63c08510 --- /dev/null +++ b/src/domains/gpdingestion-secrets/secret/itn-uat/configs.json @@ -0,0 +1,2 @@ +{ +} diff --git a/src/domains/gpdingestion-secrets/secret/itn-uat/secret.ini b/src/domains/gpdingestion-secrets/secret/itn-uat/secret.ini new file mode 100644 index 0000000000..c31c0311f6 --- /dev/null +++ b/src/domains/gpdingestion-secrets/secret/itn-uat/secret.ini @@ -0,0 +1,3 @@ +file_crypted="noedit_secret_enc.json" +kv_name="pagopa-u-itn-gpdingestion-kv" +kv_sops_key_name="pagopa-u-gpdingestion-sops-key" diff --git a/src/domains/gpdingestion-secrets/sops.sh b/src/domains/gpdingestion-secrets/sops.sh new file mode 100755 index 0000000000..347b11d0ef --- /dev/null +++ b/src/domains/gpdingestion-secrets/sops.sh @@ -0,0 +1,137 @@ +#!/bin/bash + +# set -x # Uncomment this line to enable debug mode + +# +# how to use `sh sops.sh` +# ℹ️ This script allows you to create a sops file with the relative azure key, +# it also allows you to edit the secrets and add them with the script. +# ℹ️ This script also uses an inventory file under the "./secret//secret.ini" +# directory to load environment variables. +# + +action=$1 +env=$2 +shift 2 +# shellcheck disable=SC2034 +other=( "$@" ) + +if [ -z "$action" ]; then + helpmessage=$(cat < -> decrypt json file in specified environment + example: ./sops.sh d itn-dev + example: ./sops.sh decrypt itn-dev + +./sops.sh s -> search in enc file in specified environment + example: ./sops.sh s itn-dev + example: ./sops.sh search itn-dev + +./sops.sh n -> create new file enc json template in specified environment + example: ./sops.sh n itn-dev + example: ./sops.sh new itn-dev + +./sops.sh a -> add new secret record to enc json in specified environment + example: ./sops.sh a itn-dev + example: ./sops.sh add itn-dev + +./sops.sh e -> edit enc json record in specified environment + example: ./sops.sh e itn-dev + example: ./sops.sh edit itn-dev + +./sops.sh f -> enc a json file in a specified environment + example: ./sops.sh f itn-dev + +EOF +) + echo "$helpmessage" + exit 0 +fi + +if [ -z "$env" ]; then + echo "env should be something like: itn-dev, itn-uat or itn-prod." + exit 0 +fi + +echo "🔨 Mandatory variables are correct" +file_crypted="" +kv_name="" +kv_sops_key_name="" + +# shellcheck disable=SC1090 +source "./secret/$env/secret.ini" + +echo "🔨 All variables loaded" + +# Check if kv_name and file_crypted variables are not empty +if [ -z "${kv_name}" ]; then + echo "❌ Error: kv_name variable is not defined correctly." + exit 1 +fi + +if [ -z "$file_crypted" ]; then + echo "❌ Error: file_crypted variable is not defined correctly." + exit 1 +fi + +encrypted_file_path="./secret/$env/$file_crypted" + +# Check if the key exists in the Key Vault +# shellcheck disable=SC2154 +kv_key_url=$(az keyvault key show --vault-name "$kv_name" --name "$kv_sops_key_name" --query "key.kid" -o tsv) +if [ -z "$kv_key_url" ]; then + echo "❌ The key does not exist." + exit 1 +fi +echo "[INFO] Key URL: $kv_key_url" + +echo "🔨 Key URL loaded correctly" + +if echo "d decrypt a add s search n new e edit f" | grep -w "$action" > /dev/null; then + case $action in + "d"|"decrypt") + sops --decrypt --azure-kv "$kv_key_url" "$encrypted_file_path" + if [ $? -eq 1 ]; then + echo "❌ File $encrypted_file_path NOT encrypted" + exit 0 + fi + ;; + "s"|"search") + read -r -p 'key: ' key + sops --decrypt --azure-kv "$kv_key_url" "$encrypted_file_path" | grep -i "$key" + ;; + "a"|"add") + read -r -p 'key: ' key + read -r -p 'value: ' value + sops -i --set '["'"$key"'"] "'"$value"'"' --azure-kv "$kv_key_url" "$encrypted_file_path" + echo "✅ Added key" + ;; + "n"|"new") + if [ -f "$encrypted_file_path" ]; then + echo "⚠️ file $encrypted_file_path already exists" + exit 0 + fi + echo "{}" > "$encrypted_file_path" + sops --encrypt -i --azure-kv "$kv_key_url" "$encrypted_file_path" + echo "✅ created new file for sops" + ;; + "e"|"edit") + if [ ! -f "$encrypted_file_path" ]; then + echo "⚠️ file $encrypted_file_path not found" + exit 1 + fi + + sops --azure-kv "$kv_key_url" "$encrypted_file_path" + echo "✅ edit file completed" + + ;; + "f") + read -r -p 'file: ' file + sops --encrypt --azure-kv "$kv_key_url" "./secret/$env/$file" > "$encrypted_file_path" + ;; + esac +else + echo "⚠️ Action not allowed." + exit 1 +fi diff --git a/src/domains/gpdingestion-secrets/terraform.sh b/src/domains/gpdingestion-secrets/terraform.sh new file mode 100755 index 0000000000..047a7512d0 --- /dev/null +++ b/src/domains/gpdingestion-secrets/terraform.sh @@ -0,0 +1,324 @@ +#!/bin/bash +############################################################ +# Terraform script for managing infrastructure on Azure +# Fingerprint: d2hhdHlvdXdhbnQ/Cg== +############################################################ +# Global variables +# Version format x.y accepted +vers="1.11" +script_name=$(basename "$0") +git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/azure/${script_name}" +tmp_file="${script_name}.new" +# Check if the third parameter exists and is a file +if [ -n "$3" ] && [ -f "$3" ]; then + FILE_ACTION=true +else + FILE_ACTION=false +fi + +# Define functions +function clean_environment() { + rm -rf .terraform + rm tfplan 2>/dev/null + echo "cleaned!" +} + +function download_tool() { + #default value + cpu_type="intel" + os_type=$(uname) + + # only on MacOS + if [ "$os_type" == "Darwin" ]; then + cpu_brand=$(sysctl -n machdep.cpu.brand_string) + if grep -q -i "intel" <<< "$cpu_brand"; then + cpu_type="intel" + else + cpu_type="arm" + fi + fi + + echo $cpu_type + tool=$1 + git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/golang/${tool}_${cpu_type}" + if ! command -v $tool &> /dev/null; then + if ! curl -sL "$git_repo" -o "$tool"; then + echo "Error downloading ${tool}" + return 1 + else + chmod +x $tool + echo "${tool} downloaded! Please note this tool WON'T be copied in your **/bin folder for safety reasons. +You need to do it yourself!" + read -p "Press enter to continue" + + + fi + fi +} + +function extract_resources() { + TF_FILE=$1 + ENV=$2 + TARGETS="" + + # Check if the file exists + if [ ! -f "$TF_FILE" ]; then + echo "File $TF_FILE does not exist." + exit 1 + fi + + # Check if the directory exists + if [ ! -d "./env/$ENV" ]; then + echo "Directory ./env/$ENV does not exist." + exit 1 + fi + + TMP_FILE=$(mktemp) + grep -E '^resource|^module' $TF_FILE > $TMP_FILE + + while read -r line ; do + TYPE=$(echo $line | cut -d '"' -f 1 | tr -d ' ') + if [ "$TYPE" == "module" ]; then + NAME=$(echo $line | cut -d '"' -f 2) + TARGETS+=" -target=\"$TYPE.$NAME\"" + else + NAME1=$(echo $line | cut -d '"' -f 2) + NAME2=$(echo $line | cut -d '"' -f 4) + TARGETS+=" -target=\"$NAME1.$NAME2\"" + fi + done < $TMP_FILE + + rm $TMP_FILE + + echo "./terraform.sh $action $ENV $TARGETS" +} + +function help_usage() { + echo "terraform.sh Version ${vers}" + echo + echo "Usage: ./script.sh [ACTION] [ENV] [OTHER OPTIONS]" + echo "es. ACTION: init, apply, plan, etc." + echo "es. ENV: dev, uat, prod, etc." + echo + echo "Available actions:" + echo " clean Remove .terraform* folders and tfplan files" + echo " help This help" + echo " list List every environment available" + echo " update Update this script if possible" + echo " summ Generate summary of Terraform plan" + echo " tflist Generate an improved output of terraform state list" + echo " tlock Generate or update the dependency lock file" + echo " * any terraform option" +} + +function init_terraform() { + if [ -n "$env" ]; then + terraform init -reconfigure -backend-config="./env/$env/backend.tfvars" + else + echo "ERROR: no env configured!" + exit 1 + fi +} + +function list_env() { + # Check if env directory exists + if [ ! -d "./env" ]; then + echo "No environment directory found" + exit 1 + fi + + # List subdirectories under env directory + env_list=$(ls -d ./env/*/ 2>/dev/null) + + # Check if there are any subdirectories + if [ -z "$env_list" ]; then + echo "No environments found" + exit 1 + fi + + # Print the list of environments + echo "Available environments:" + for env in $env_list; do + env_name=$(echo "$env" | sed 's#./env/##;s#/##') + echo "- $env_name" + done +} + +function other_actions() { + if [ -n "$env" ] && [ -n "$action" ]; then + terraform "$action" -var-file="./env/$env/terraform.tfvars" -compact-warnings $other + else + echo "ERROR: no env or action configured!" + exit 1 + fi +} + +function state_output_taint_actions() { + if [ "$action" == "tflist" ]; then + # If 'tflist' is not installed globally and there is no 'tflist' file in the current directory, + # attempt to download the 'tflist' tool + if ! command -v tflist &> /dev/null && [ ! -f "tflist" ]; then + download_tool "tflist" + if [ $? -ne 0 ]; then + echo "Error: Failed to download tflist!!" + exit 1 + else + echo "tflist downloaded!" + fi + fi + if command -v tflist &> /dev/null; then + terraform state list | tflist + else + terraform state list | ./tflist + fi + else + terraform $action $other + fi +} + + +function parse_tfplan_option() { + # Create an array to contain arguments that do not start with '-tfplan=' + local other_args=() + + # Loop over all arguments + for arg in "$@"; do + # If the argument starts with '-tfplan=', extract the file name + if [[ "$arg" =~ ^-tfplan= ]]; then + echo "${arg#*=}" + else + # If the argument does not start with '-tfplan=', add it to the other_args array + other_args+=("$arg") + fi + done + + # Print all arguments in other_args separated by spaces + echo "${other_args[@]}" +} + +function tfsummary() { + local plan_file + plan_file=$(parse_tfplan_option "$@") + if [ -z "$plan_file" ]; then + plan_file="tfplan" + fi + action="plan" + other="-out=${plan_file}" + other_actions + if [ -n "$(command -v tf-summarize)" ]; then + tf-summarize -tree "${plan_file}" + else + echo "tf-summarize is not installed" + fi + if [ "$plan_file" == "tfplan" ]; then + rm $plan_file + fi +} + +function update_script() { + # Check if the repository was cloned successfully + if ! curl -sL "$git_repo" -o "$tmp_file"; then + echo "Error cloning the repository" + rm "$tmp_file" 2>/dev/null + return 1 + fi + + # Check if a newer version exists + remote_vers=$(sed -n '8s/vers="\(.*\)"/\1/p' "$tmp_file") + if [ "$(printf '%s\n' "$vers" "$remote_vers" | sort -V | tail -n 1)" == "$vers" ]; then + echo "The local script version is equal to or newer than the remote version." + rm "$tmp_file" 2>/dev/null + return 0 + fi + + # Check the fingerprint + local_fingerprint=$(sed -n '4p' "$0") + remote_fingerprint=$(sed -n '4p' "$tmp_file") + + if [ "$local_fingerprint" != "$remote_fingerprint" ]; then + echo "The local and remote file fingerprints do not match." + rm "$tmp_file" 2>/dev/null + return 0 + fi + + # Show the current and available versions to the user + echo "Current script version: $vers" + echo "Available script version: $remote_vers" + + # Ask the user if they want to update the script + read -rp "Do you want to update the script to version $remote_vers? (y/n): " answer + + if [ "$answer" == "y" ] || [ "$answer" == "Y" ]; then + # Replace the local script with the updated version + cp "$tmp_file" "$script_name" + chmod +x "$script_name" + rm "$tmp_file" 2>/dev/null + + echo "Script successfully updated to version $remote_vers" + else + echo "Update canceled by the user" + fi + + rm "$tmp_file" 2>/dev/null +} + +# Check arguments number +if [ "$#" -lt 1 ]; then + help_usage + exit 0 +fi + +# Parse arguments +action=$1 +env=$2 +filetf=$3 +shift 2 +other=$@ + +if [ -n "$env" ]; then + # shellcheck source=/dev/null + source "./env/$env/backend.ini" + if [ -z "$(command -v az)" ]; then + echo "az not found, cannot proceed" + exit 1 + fi + az account set -s "${subscription}" +fi + +# Call appropriate function based on action +case $action in + clean) + clean_environment + ;; + ?|help|-h) + help_usage + ;; + init) + init_terraform "$other" + ;; + list) + list_env + ;; + output|state|taint|tflist) + init_terraform + state_output_taint_actions $other + ;; + summ) + init_terraform + tfsummary "$other" + ;; + tlock) + terraform providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=darwin_arm64 -platform=linux_amd64 + ;; + update) + update_script + ;; + *) + if [ "$FILE_ACTION" = true ]; then + extract_resources "$filetf" "$env" + else + init_terraform + other_actions "$other" + fi + ;; +esac diff --git a/src/domains/gpdingestion-secrets/terrasops.sh b/src/domains/gpdingestion-secrets/terrasops.sh new file mode 100644 index 0000000000..32be3bd04f --- /dev/null +++ b/src/domains/gpdingestion-secrets/terrasops.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# set -x # Uncomment this line to enable debug mode + +# +# ℹ️ This script is used by terraform, to decrypt all secrets on sops and export them to json. +# This way it can loop through them and use them to insert them inside the KV +# ⚠️ Do not add additional echos to the script in case of golden path, +# as the script only needs to return a json +# + +eval "$(jq -r '@sh "export terrasops_env=\(.env)"')" + +# shellcheck disable=SC1090 +source "./secret/$terrasops_env/secret.ini" +encrypted_file_path="./secret/$terrasops_env/$file_crypted" + +if [ -f "$encrypted_file_path" ]; then + # Load the values of azure_kv.vault_url and azure_kv.name from the JSON file + azure_kv_vault_url=$(jq -r '.sops.azure_kv[0].vault_url' "$encrypted_file_path") + azure_kv_name=$(jq -r '.sops.azure_kv[0].name' "$encrypted_file_path") + + if [ -z "$azure_kv_vault_url" ] || [ -z "$azure_kv_name" ]; then + echo "❌ Error: Unable to load the values of azure_kv.vault_url and azure_kv.name from the JSON file" >&2 + exit 1 + fi + sops -d --azure-kv "azure_kv_vault_url" "$encrypted_file_path" | jq -c +else + echo "{}" | jq -c +fi From 0e41c0ae2004b4bac585b2996541ae66918ee528 Mon Sep 17 00:00:00 2001 From: acialini Date: Thu, 24 Oct 2024 09:51:35 +0200 Subject: [PATCH 04/55] [PPANTT-168] feat: Introducing debezium connector, updated module version --- src/domains/gpdingestion-app/00_data.tf | 16 +- src/domains/gpdingestion-app/02_namespace.tf | 28 ++-- .../03_serviceaccounts_azure_devops.tf | 2 +- .../gpdingestion-app/05_debezium_connect.tf | 139 ++++++++++++++++++ src/domains/gpdingestion-app/99_main.tf | 14 +- src/domains/gpdingestion-app/99_variables.tf | 95 ++++++++++++ .../env/itn-dev/terraform.tfvars | 17 +++ .../env/itn-prod/terraform.tfvars | 17 +++ .../env/itn-uat/terraform.tfvars | 17 +++ .../gpdingestion-app/set_registry_secrets.sh | 27 ++++ .../gpdingestion-app/yaml/debezium-rbac.yaml | 13 ++ .../gpdingestion-app/yaml/debezium-role.yaml | 10 ++ .../yaml/debezium-secrets.yaml | 9 ++ .../gpdingestion-app/yaml/kafka-connect.yaml | 55 +++++++ .../yaml/postgres-connector.yaml | 25 ++++ .../gpdingestion-app/yaml/zookeeper.yaml | 26 ++++ .../gpdingestion-common/03_eventhub.tf | 26 ++-- .../gpdingestion-common/10_github_identity.tf | 6 +- .../gpdingestion-secrets/01_keyvault.tf | 2 +- src/domains/gpdingestion-secrets/99_main.tf | 14 +- 20 files changed, 520 insertions(+), 38 deletions(-) create mode 100644 src/domains/gpdingestion-app/05_debezium_connect.tf create mode 100644 src/domains/gpdingestion-app/set_registry_secrets.sh create mode 100644 src/domains/gpdingestion-app/yaml/debezium-rbac.yaml create mode 100644 src/domains/gpdingestion-app/yaml/debezium-role.yaml create mode 100644 src/domains/gpdingestion-app/yaml/debezium-secrets.yaml create mode 100644 src/domains/gpdingestion-app/yaml/kafka-connect.yaml create mode 100644 src/domains/gpdingestion-app/yaml/postgres-connector.yaml create mode 100644 src/domains/gpdingestion-app/yaml/zookeeper.yaml diff --git a/src/domains/gpdingestion-app/00_data.tf b/src/domains/gpdingestion-app/00_data.tf index f05bb6d119..fc13f587b2 100644 --- a/src/domains/gpdingestion-app/00_data.tf +++ b/src/domains/gpdingestion-app/00_data.tf @@ -1,9 +1,17 @@ ### EVH -resource "azurerm_eventhub_authorization_rule" "cdc_connection_string" { +data "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" { name = "cdc-connection-string" namespace_name = "${local.project}-evh" resource_group_name = "${local.project}-evh-rg" - listen = true - send = true - manage = true +} + +data "azurerm_eventhub_namespace" "eventhub" { + name = "${local.project}-evh" + namespace_name = "${local.project}-evh" + resource_group_name = "${local.project}-evh-rg" +} + +data "azurerm_postgresql_database" "apd_db" { + name = var.postgres_db_name + resource_group_name = "${local.project}-evh-rg" } diff --git a/src/domains/gpdingestion-app/02_namespace.tf b/src/domains/gpdingestion-app/02_namespace.tf index a62b8cc66c..759fa6152b 100644 --- a/src/domains/gpdingestion-app/02_namespace.tf +++ b/src/domains/gpdingestion-app/02_namespace.tf @@ -4,17 +4,25 @@ resource "kubernetes_namespace" "namespace" { } } -module "pod_identity" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_pod_identity?ref=v8.18.0" +module "workload_identity" { + source = "./.terraform/modules/__v3__/kubernetes_workload_identity_init" - resource_group_name = local.aks_resource_group_name - location = var.location - tenant_id = data.azurerm_subscription.current.tenant_id - cluster_name = local.aks_name + workload_identity_name_prefix = "${var.domain}-workload-identity" + workload_identity_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name + workload_identity_location = var.location +} + +module "workload_identity" { + source = "./.terraform/modules/__v3__/kubernetes_workload_identity_configuration" - identity_name = "${kubernetes_namespace.namespace.metadata[0].name}-pod-identity" - namespace = kubernetes_namespace.namespace.metadata[0].name - key_vault_id = data.azurerm_key_vault.kv.id + workload_identity_name_prefix = "${var.domain}-poc" + workload_identity_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name + aks_name = data.azurerm_kubernetes_cluster.aks.name + aks_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name + namespace = var.domain - secret_permissions = ["Get"] + key_vault_id = data.azurerm_key_vault.kv.id + key_vault_certificate_permissions = ["Get"] + key_vault_key_permissions = ["Get"] + key_vault_secret_permissions = ["Get"] } diff --git a/src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf b/src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf index f24964a97d..0b297b7fdf 100644 --- a/src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf +++ b/src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf @@ -5,7 +5,7 @@ resource "kubernetes_namespace" "namespace_system" { } module "kubernetes_service_account" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_service_account?ref=v8.18.0" + source = "./.terraform/modules/__v3__/kubernetes_service_account?ref=v8.18.0" name = "azure-devops" namespace = "${var.domain}-system" } diff --git a/src/domains/gpdingestion-app/05_debezium_connect.tf b/src/domains/gpdingestion-app/05_debezium_connect.tf new file mode 100644 index 0000000000..6d8bf13ccd --- /dev/null +++ b/src/domains/gpdingestion-app/05_debezium_connect.tf @@ -0,0 +1,139 @@ +data "azurerm_key_vault_secret" "pgres_admin_login" { + name = "db-apd-user-name" + key_vault_id = "pagopa-${var.env_short}-gps-kv" +} + +data "azurerm_key_vault_secret" "pgres_admin_pwd" { + name = "db-apd-user-password" + key_vault_id = "pagopa-${var.env_short}-gps-kv" +} + +resource "helm_release" "strimzi-kafka-operator" { + name = "strimzi-kafka-operator" + repository = "https://strimzi.io/charts/strimzi-kafka-operator" + chart = "strimzi-kafka-operator" + version = "0.8.2" + + namespace = kubernetes_namespace.namespace.metadata[0].name +} + +locals { + + debezium_role_yaml = templatefile("${path.module}/yaml/debezium-role.yaml", { + namespace = kubernetes_namespace.namespace.metadata[0].name + }) + + debezium_rbac_yaml = templatefile("${path.module}/yaml/debezium-rbac.yaml", { + namespace = kubernetes_namespace.namespace.metadata[0].name + }) + + debezium_secrets_yaml = templatefile("${path.module}/yaml/debezium-secretes.yaml", { + namespace = kubernetes_namespace.namespace.metadata[0].name + username = data.azurerm_key_vault_secret.pgres_admin_login.value + password = data.azurerm_key_vault_secret.pgres_admin_pwd.value + }) + + zookeeper_yaml = templatefile("${path.module}/yaml/zookeper.yaml", { + namespace = kubernetes_namespace.namespace.metadata[0].name + zookeeper_replicas = var.zookeeper_replicas + zookeeper_request_memory = var.zookeeper_request_memory + zookeeper_request_cpu = var.zookeeper_request_cpu + zookeeper_limits_memory = var.zookeeper_limits_memory + zookeeper_limits_cpu = var.zookeeper_limits_cpu + zookeeper_jvm_xms = var.zookeeper_jvm_xms + zookeeper_jvm_xmx = var.zookeeper_jvm_xmx + zookeeper_storage_size = var.zookeeper_storage_size + }) + + kafka_connect_yaml = templatefile("${path.module}/yaml/kafka-connect.yaml", { + namespace = kubernetes_namespace.namespace.metadata[0].name + replicas = var.replicas + request_memory = var.request_memory + request_cpu = var.request_cpu + limits_memory = var.limits_memory + limits_cpu = var.limits_cpu + bootstrap_servers = "pagopa-${var.env_short}-${var.location_short}-${local.project}-evh.servicebus.windows.net:9092" + eh_connection_string = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string}\";" + container_registry = var.container_registry + }) + + postgres_connector_yaml = templatefile("${path.module}/yaml/postgres-connector.yaml", { + namespace = kubernetes_namespace.namespace.metadata[0].name + postgres_hostname = "pagopa-${var.env_short}-gpd-postgresql.postgres.database.azure.com" + postgres_port = 6432 + postgres_db_name = var.postgres_db_name + postgres_topic_prefix = "gpd" + tasks_max = var.tasks_max + }) + +} + +resource "kubectl_manifest" "debezium_role" { + force_conflicts = true + yaml_body = local.debezium_role_yaml +} + +resource "kubectl_manifest" "debezium_secrets" { + force_conflicts = true + yaml_body = local.debezium_secrets_yaml +} + +resource "kubectl_manifest" "debezoum_rbac" { + depends_on = [kubectl_manifest.debezium_role, kubectl_manifest.debezium_secrets] + force_conflicts = true + yaml_body = local.debezium_rbac_yaml +} + +resource "kubectl_manifest" "zookeper_manifest" { + depends_on = [ + helm_release.strimzi-kafka-operator + ] + force_conflicts = true + yaml_body = local.zookeeper_yaml +} + +resource "null_resource" "wait_zookeeper" { + depends_on = [ + kubectl_manifest.zookeper_manifest + ] + provisioner "local-exec" { + command = "while [ true ]; do STATUS=`kubectl -n ${kubernetes_namespace.namespace.metadata[0].name} get Kafka -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Zookeper SUCCEEDED\" ; break ; else echo \"Zookeeper INPROGRESS\"; sleep 3; fi ; done" + interpreter = ["/bin/bash", "-c"] + } +} + +resource "kubectl_manifest" "kafka_connect" { + depends_on = [ + helm_release.strimzi-kafka-operator + ] + force_conflicts = true + yaml_body = local.kafka_connect_yaml +} + +resource "null_resource" "wait_kafka_connect" { + depends_on = [ + kubectl_manifest.kafka_connect + ] + provisioner "local-exec" { + command = "while [ true ]; do STATUS=`kubectl -n ${kubernetes_namespace.namespace.metadata[0].name} get KafkaConnect -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Kafka Connect SUCCEEDED\" ; break ; else echo \"Kafka Connect INPROGRESS\"; sleep 3; fi ; done" + interpreter = ["/bin/bash", "-c"] + } +} + +resource "kubectl_manifest" "postgres_connector" { + depends_on = [ + helm_release.strimzi-kafka-operator + ] + force_conflicts = true + yaml_body = local.postgres_connector_yaml +} + +resource "null_resource" "wait_postgres_connector" { + depends_on = [ + kubectl_manifest.kafka_connect + ] + provisioner "local-exec" { + command = "while [ true ]; do STATUS=`kubectl -n ${kubernetes_namespace.namespace.metadata[0].name} get KafkaConnector -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Postgres Connector SUCCEEDED\" ; break ; else echo \"Postgres Connector INPROGRESS\"; sleep 3; fi ; done" + interpreter = ["/bin/bash", "-c"] + } +} diff --git a/src/domains/gpdingestion-app/99_main.tf b/src/domains/gpdingestion-app/99_main.tf index 8bf0b91ba8..3ac065d2cd 100644 --- a/src/domains/gpdingestion-app/99_main.tf +++ b/src/domains/gpdingestion-app/99_main.tf @@ -3,23 +3,23 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "<= 3.106.0" + version = "<= 3.116.0" } azuread = { source = "hashicorp/azuread" - version = "<= 2.47.0" + version = "<= 3.0.2" } null = { source = "hashicorp/null" - version = "<= 3.2.1" + version = "<= 3.2.3" } kubernetes = { source = "hashicorp/kubernetes" - version = "<= 2.29.0" + version = "<= 2.33.0" } helm = { source = "hashicorp/helm" - version = "<= 2.12.1" + version = "<= 2.16.0" } } @@ -47,3 +47,7 @@ provider "helm" { config_path = "${var.k8s_kube_config_path_prefix}/config-${local.aks_name}" } } + +module "__v3__" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" +} diff --git a/src/domains/gpdingestion-app/99_variables.tf b/src/domains/gpdingestion-app/99_variables.tf index e49b5c9cb7..410c72ea94 100644 --- a/src/domains/gpdingestion-app/99_variables.tf +++ b/src/domains/gpdingestion-app/99_variables.tf @@ -156,3 +156,98 @@ variable "pod_disruption_budgets" { description = "Pod disruption budget for domain namespace" default = {} } + +variable "zookeeper_replicas" { + type = number + description = "Zookeeper Replicas" + default = 1 +} + +variable "zookeeper_request_memory" { + type = string + description = "Zookeeper Request Memory" + default = "512m" +} + +variable "zookeeper_request_cpu" { + type = string + description = "Zookeeper Request CPU" + default = "0.5" +} + +variable "zookeeper_limits_memory" { + type = string + description = "Zookeeper Limit Memory" + default = "512mi" +} + +variable "zookeeper_limits_cpu" { + type = string + description = "Zookeeper Limit CPU" + default = "0.5" +} + +variable "zookeeper_jvm_xms" { + type = string + description = "Zookeeper Jvm Xms" + default = "512mi" +} + +variable "zookeeper_jvm_xmx" { + type = string + description = "Zookeeper Jvm Xmx" + default = "512mi" +} + +variable "zookeeper_storage_size" { + type = string + description = "Zookeeper Storage Size" + default = "100Gi" +} + +variable "container_registry" { + type = string + description = "Container Registry" +} + +variable "postgres_db_name" { + type = string + description = "Postgres Database Name" + default = "apd" +} + +variable "tasks_max" { + type = string + description = "Number of tasks" + default = "1" +} + +variable "replicas" { + type = number + description = "Number of replicas in cluster" + default = 1 +} + +variable "request_memory" { + type = string + description = "Connect Request Memory" + default = "512m" +} + +variable "request_cpu" { + type = string + description = "Connect Request CPU" + default = "0.5" +} + +variable "limits_memory" { + type = string + description = "Connect Limit Memory" + default = "512mi" +} + +variable "limits_cpu" { + type = string + description = "Connect Limit CPU" + default = "0.5" +} diff --git a/src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars b/src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars index 880d882c52..af815d1ab0 100644 --- a/src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars +++ b/src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars @@ -36,3 +36,20 @@ ingress_load_balancer_ip = "10.3.2.250" is_feature_enabled = { gpdingestion = true } + +zookeeper_replicas = 1 +zookeeper_request_memory = "512mi" +zookeeper_request_cpu = "0.5" +zookeeper_limits_memory = "512mi" +zookeeper_limits_cpu = "0.5" +zookeeper_jvm_xms = "512mi" +zookeeper_jvm_xmx = "512mi" +zookeeper_storage_size = "100Gi" +replicas = 1 +request_cpu = "0.5" +request_memory = "512mi" +limits_memory = "512mi" +limits_cpu = "0.5" +postgres_db_name = "apd" +tasks_max = "1" +container_registry = "TBD" diff --git a/src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars b/src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars index 877f59b559..12fe7aeef0 100644 --- a/src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars +++ b/src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars @@ -45,3 +45,20 @@ pod_disruption_budgets = { } }, } + +zookeeper_replicas = 3 +zookeeper_request_memory = "1024mi" +zookeeper_request_cpu = "1" +zookeeper_limits_memory = "1024mi" +zookeeper_limits_cpu = "1" +zookeeper_jvm_xms = "1024mi" +zookeeper_jvm_xmx = "1024mi" +zookeeper_storage_size = "100Gi" +replicas = 3 +request_cpu = "1" +request_memory = "512mi" +limits_memory = "1024mi" +limits_cpu = "1" +postgres_db_name = "apd" +tasks_max = "5" +container_registry = "TBD" diff --git a/src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars b/src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars index 2ece4eb2a7..6ceb56d371 100644 --- a/src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars +++ b/src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars @@ -36,3 +36,20 @@ ingress_load_balancer_ip = "10.3.2.250" is_feature_enabled = { gpdingestion = true } + +zookeeper_replicas = 3 +zookeeper_request_memory = "1024mi" +zookeeper_request_cpu = "1" +zookeeper_limits_memory = "1024mi" +zookeeper_limits_cpu = "1" +zookeeper_jvm_xms = "1024mi" +zookeeper_jvm_xmx = "1024mi" +zookeeper_storage_size = "100Gi" +replicas = 3 +request_cpu = "1" +request_memory = "512mi" +limits_memory = "1024mi" +limits_cpu = "1" +postgres_db_name = "apd" +tasks_max = "5" +container_registry = "TBD" diff --git a/src/domains/gpdingestion-app/set_registry_secrets.sh b/src/domains/gpdingestion-app/set_registry_secrets.sh new file mode 100644 index 0000000000..74212c7f55 --- /dev/null +++ b/src/domains/gpdingestion-app/set_registry_secrets.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Check if the required parameters are provided +if [ "$#" -ne 4 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Assign parameters to variables +DOCKER_SERVER=$1 +DOCKER_USERNAME=$2 +DOCKER_PASSWORD=$3 +DOCKER_EMAIL=$4 + +# Create the Docker registry secret using kubectl +kubectl create secret docker-registry registry-credential \ + --docker-server="$DOCKER_SERVER" \ + --docker-username="$DOCKER_USERNAME" \ + --docker-password="$DOCKER_PASSWORD" \ + --docker-email="$DOCKER_EMAIL" + +# Check if the secret was created successfully +if [ $? -eq 0 ]; then + echo "Docker registry secret 'registry-credential' created successfully." +else + echo "Failed to create Docker registry secret." +fi diff --git a/src/domains/gpdingestion-app/yaml/debezium-rbac.yaml b/src/domains/gpdingestion-app/yaml/debezium-rbac.yaml new file mode 100644 index 0000000000..1156ec0b80 --- /dev/null +++ b/src/domains/gpdingestion-app/yaml/debezium-rbac.yaml @@ -0,0 +1,13 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: connector-configuration-role-binding + namespace: ${namespace} +subjects: + - kind: ServiceAccount + name: debezium-connect-cluster-connect + namespace: ${namespace} +roleRef: + kind: Role + name: connector-configuration-role + apiGroup: rbac.authorization.k8s.io diff --git a/src/domains/gpdingestion-app/yaml/debezium-role.yaml b/src/domains/gpdingestion-app/yaml/debezium-role.yaml new file mode 100644 index 0000000000..15f47655c3 --- /dev/null +++ b/src/domains/gpdingestion-app/yaml/debezium-role.yaml @@ -0,0 +1,10 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: connector-configuration-role + namespace: ${namespace} +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["debezium-secret", "registry-credential"] + verbs: ["get"] diff --git a/src/domains/gpdingestion-app/yaml/debezium-secrets.yaml b/src/domains/gpdingestion-app/yaml/debezium-secrets.yaml new file mode 100644 index 0000000000..4d996e3e90 --- /dev/null +++ b/src/domains/gpdingestion-app/yaml/debezium-secrets.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: debezium-secret + namespace: ${namespace} +type: Opaque +data: + username: ${username} + password: ${password} diff --git a/src/domains/gpdingestion-app/yaml/kafka-connect.yaml b/src/domains/gpdingestion-app/yaml/kafka-connect.yaml new file mode 100644 index 0000000000..fe47c2f503 --- /dev/null +++ b/src/domains/gpdingestion-app/yaml/kafka-connect.yaml @@ -0,0 +1,55 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnect +metadata: + name: debezium-connect-cluster + namespace: ${namespace} + annotations: + strimzi.io/use-connector-resources: "true" +spec: + version: 3.1.0 + replicas: ${replicas} + bootstrapServers: ${bootstrap_servers} + resources: + requests: + memory: $connect_request_memory} + cpu: ${connect_request_cpu} + limits: + memory: ${connect_limits_memory} + cpu: ${connect_limits_cpu} + config: + config.providers: secrets + config.providers.secrets.class: io.strimzi.kafka.KubernetesSecretConfigProvider + group.id: connect-cluster + offset.storage.topic: connect-cluster-offsets + config.storage.topic: connect-cluster-configs + status.storage.topic: connect-cluster-status + # -1 means it will use the default replication factor configured in the broker + config.storage.replication.factor: 1 + offset.storage.replication.factor: 1 + status.storage.replication.factor: 1 + rest.advertised.host.name: connect + offset.flush.interval.ms: 10000 + key.converter: org.apache.kafka.connect.json.JsonConverter + value.converter: org.apache.kafka.connect.json.JsonConverter + internal.key.converter: org.apache.kafka.connect.json.JsonConverter + internal.value.converter: org.apache.kafka.connect.json.JsonConverter + internal.key.converter.schemas.enable: false + internal.value.converter.schemas.enable: false + security.protocol: SASL_SSL + sasl.mechanism: PLAIN + sasl.jaas.config: ${eh_connection_string} + producer.security.protocol: SASL_SSL + producer.sasl.mechanism: PLAIN + producer.sasl.jaas.config: ${eh_connection_string} + consumer.security.protocol: SASL_SSL + consumer.sasl.mechanism: PLAIN + consumer.sasl.jaas.config: ${eh_connection_string} + build: + output: + type: docker + image: ${container_registry}/debezium-connector-postgres:latest + plugins: + - name: debezium-postgres-connector + artifacts: + - type: tgz + url: https://repo1.maven.org/maven2/io/debezium/debezium-connector-postgres/3.0.0.Final/debezium-connector-postgres-3.0.0.Final-plugin.tar.gz diff --git a/src/domains/gpdingestion-app/yaml/postgres-connector.yaml b/src/domains/gpdingestion-app/yaml/postgres-connector.yaml new file mode 100644 index 0000000000..26048026a3 --- /dev/null +++ b/src/domains/gpdingestion-app/yaml/postgres-connector.yaml @@ -0,0 +1,25 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnector +metadata: + name: debezium-connector-postgres + namespace: ${namespace} + labels: + strimzi.io/cluster: debezium-connect-cluster +spec: + class: io.debezium.connector.postgresql.PostgresConnector + tasksMax: ${tasks_max} + config: + connector.class: "io.debezium.connector.postgresql.PostgresConnector", + key.converter: "org.apache.kafka.connect.json.JsonConverter", + key.converter.schemas.enable: false, + value.converter: "org.apache.kafka.connect.json.JsonConverter", + value.converter.schemas.enable: false, + tasks.max: ${tasks_max}, + database.hostname: ${postgres_hostname}, + database.port: {postgres_port}, + database.user: ${secrets:${namespace}/debezium-secret:username}, + database.password: ${secrets:${namespace}/debezium-secret:password}, + database.dbname: ${postgres_db_name}, + topic.prefix: ${postgres_topic_prefix}, + schema.include.list: "apd", + table.include.list: "apd.payment_option,apd.payment_option_metadata,apd.payment_position,apd.transfer,apd.transfer_metadata" diff --git a/src/domains/gpdingestion-app/yaml/zookeeper.yaml b/src/domains/gpdingestion-app/yaml/zookeeper.yaml new file mode 100644 index 0000000000..7bbbe7b256 --- /dev/null +++ b/src/domains/gpdingestion-app/yaml/zookeeper.yaml @@ -0,0 +1,26 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: Kafka +metadata: + name: debezium-connect-cluster + namespace: ${namespace} +spec: + zookeeper: + replicas: ${zookeeper_replicas} + logging: + type: inline + loggers: + zookeeper.root.logger: INFO + resources: + requests: + memory: ${zookeeper_request_memory} + cpu: ${zookeeper_request_cpu} + limits: + memory: ${zookeeper_limits_memory} + cpu: ${zookeeper_limits_cpu} + jvmOptions: + -Xms: ${zookeeper_jvm_xms} + -Xmx: ${zookeeper_jvm_xmx} + storage: + type: persistent-claim + size: ${zookeeper_storage_size} + diff --git a/src/domains/gpdingestion-common/03_eventhub.tf b/src/domains/gpdingestion-common/03_eventhub.tf index 98cc499115..46a7ce1c4e 100644 --- a/src/domains/gpdingestion-common/03_eventhub.tf +++ b/src/domains/gpdingestion-common/03_eventhub.tf @@ -6,7 +6,7 @@ resource "azurerm_resource_group" "eventhub_ita_rg" { } module "eventhub_namespace" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub?ref=v8.22.0" + source = "./.terraform/modules/__v3__/eventhub_configuration" name = "${local.project}-evh" location = var.location resource_group_name = azurerm_resource_group.eventhub_ita_rg.name @@ -51,13 +51,21 @@ module "eventhub_namespace" { # # CONFIGURATION # -module "eventhub_gpdingestion_configuration" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration?ref=v8.22.0" - count = var.is_feature_enabled.eventhub ? 1 : 0 - - event_hub_namespace_name = module.eventhub_namespace.name - event_hub_namespace_resource_group_name = azurerm_resource_group.eventhub_ita_rg.name +#module "eventhub_gpdingestion_configuration" { +# source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration?ref=v8.22.0" +# count = var.is_feature_enabled.eventhub ? 1 : 0 +# +# event_hub_namespace_name = module.eventhub_namespace.name +# event_hub_namespace_resource_group_name = azurerm_resource_group.eventhub_ita_rg.name +# +# eventhubs = [] +#} - eventhubs = [] +resource "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" { + name = "cdc-connection-string" + namespace_name = "${local.project}-evh" + resource_group_name = "${local.project}-evh-rg" + listen = true + send = true + manage = true } - diff --git a/src/domains/gpdingestion-common/10_github_identity.tf b/src/domains/gpdingestion-common/10_github_identity.tf index 4f63c95a2e..d59a9d9cbe 100644 --- a/src/domains/gpdingestion-common/10_github_identity.tf +++ b/src/domains/gpdingestion-common/10_github_identity.tf @@ -59,7 +59,7 @@ locals { # create a module for each 20 repos module "identity_cd_01" { - source = "github.com/pagopa/terraform-azurerm-v3//github_federated_identity?ref=v8.22.0" + source = "./.terraform/modules/__v3__/github_federated_identity" # pagopa---github--identity prefix = var.prefix env_short = var.env_short @@ -131,7 +131,7 @@ resource "null_resource" "github_runner_app_permissions_to_namespace_cd_01" { # create a module for each 20 repos module "identity_pr_01" { - source = "github.com/pagopa/terraform-azurerm-v3//github_federated_identity?ref=v8.22.0" + source = "./.terraform/modules/__v3__/github_federated_identity" prefix = var.prefix env_short = var.env_short domain = "${var.domain}-01-pr" @@ -170,7 +170,7 @@ resource "azurerm_key_vault_access_policy" "gha_pr_iac_managed_identities" { # create a module for each 20 repos module "identity_ref_01" { - source = "github.com/pagopa/terraform-azurerm-v3//github_federated_identity?ref=v8.36.1" + source = "./.terraform/modules/__v3__/github_federated_identity" prefix = var.prefix env_short = var.env_short domain = "${var.domain}-01-ref" diff --git a/src/domains/gpdingestion-secrets/01_keyvault.tf b/src/domains/gpdingestion-secrets/01_keyvault.tf index e6a4a6322b..be275a9653 100644 --- a/src/domains/gpdingestion-secrets/01_keyvault.tf +++ b/src/domains/gpdingestion-secrets/01_keyvault.tf @@ -6,7 +6,7 @@ resource "azurerm_resource_group" "sec_rg" { } module "key_vault" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//key_vault?ref=v8.22.0" + source = "./.terraform/modules/__v3__/key_vault" name = "${local.product}-${var.location_short}-${var.domain}-kv" location = azurerm_resource_group.sec_rg.location diff --git a/src/domains/gpdingestion-secrets/99_main.tf b/src/domains/gpdingestion-secrets/99_main.tf index 93ec8f61ca..ba2156d6dd 100644 --- a/src/domains/gpdingestion-secrets/99_main.tf +++ b/src/domains/gpdingestion-secrets/99_main.tf @@ -2,23 +2,23 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "<= 3.106.0" + version = "<= 3.116.0" } azuread = { source = "hashicorp/azuread" - version = "<= 2.47.0" + version = "<= 3.0.2" } null = { source = "hashicorp/null" - version = "<= 3.2.1" + version = "<= 3.2.3" } external = { source = "hashicorp/external" - version = "<= 2.2.3" + version = "<= 2.3.4" } kubernetes = { source = "hashicorp/kubernetes" - version = "<= 2.16.1" + version = "<= 2.33.0" } } @@ -41,3 +41,7 @@ provider "kubernetes" { data "azurerm_subscription" "current" {} data "azurerm_client_config" "current" {} + +module "__v3__" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" +} From 9c9ce98b125e374fd52fa6d3a522f1666e4c672f Mon Sep 17 00:00:00 2001 From: acialini Date: Fri, 25 Oct 2024 09:20:01 +0200 Subject: [PATCH 05/55] [PPANTT-168] feat: updated eventhub config and kafka-connect.yaml --- .../gpdingestion-app/yaml/kafka-connect.yaml | 1 + .../gpdingestion-common/03_eventhub.tf | 153 ++++++++++++++++-- 2 files changed, 142 insertions(+), 12 deletions(-) diff --git a/src/domains/gpdingestion-app/yaml/kafka-connect.yaml b/src/domains/gpdingestion-app/yaml/kafka-connect.yaml index fe47c2f503..fdc4745ff3 100644 --- a/src/domains/gpdingestion-app/yaml/kafka-connect.yaml +++ b/src/domains/gpdingestion-app/yaml/kafka-connect.yaml @@ -29,6 +29,7 @@ spec: status.storage.replication.factor: 1 rest.advertised.host.name: connect offset.flush.interval.ms: 10000 + topic.creation.enable: false key.converter: org.apache.kafka.connect.json.JsonConverter value.converter: org.apache.kafka.connect.json.JsonConverter internal.key.converter: org.apache.kafka.connect.json.JsonConverter diff --git a/src/domains/gpdingestion-common/03_eventhub.tf b/src/domains/gpdingestion-common/03_eventhub.tf index 46a7ce1c4e..0592919026 100644 --- a/src/domains/gpdingestion-common/03_eventhub.tf +++ b/src/domains/gpdingestion-common/03_eventhub.tf @@ -48,18 +48,147 @@ module "eventhub_namespace" { tags = var.tags } -# -# CONFIGURATION -# -#module "eventhub_gpdingestion_configuration" { -# source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration?ref=v8.22.0" -# count = var.is_feature_enabled.eventhub ? 1 : 0 -# -# event_hub_namespace_name = module.eventhub_namespace.name -# event_hub_namespace_resource_group_name = azurerm_resource_group.eventhub_ita_rg.name -# -# eventhubs = [] -#} + + CONFIGURATION + +module "eventhub_gpdingestion_configuration" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration?ref=v8.22.0" + count = var.is_feature_enabled.eventhub ? 1 : 0 + + event_hub_namespace_name = module.eventhub_namespace.name + event_hub_namespace_resource_group_name = azurerm_resource_group.eventhub_ita_rg.name + + eventhubs = [ + { + name = "connect-cluster-offsets" + partitions = 1 + message_retention = 1 + consumers = [ + "connect-cluster-offsets", + ] + keys = [ + { + name = "connect-cluster-offsets" + listen = true + send = true + manage = false + } + ] + }, + { + name = "connect-cluster-status" + partitions = 1 + message_retention = 1 + consumers = [ + "connect-cluster-offsets", + ] + keys = [ + { + name = "connect-cluster-status" + listen = true + send = true + manage = false + } + ] + }, + { + name = "connect-cluster-configs" + partitions = 1 + message_retention = 1 + consumers = [ + "connect-cluster-configs", + ] + keys = [ + { + name = "connect-cluster-configs" + listen = true + send = true + manage = false + } + ] + }, + { + name = "${var.prefix}-${var.domain}.apd.payment_option" + partitions = 1 + message_retention = 1 + consumers = [ + "${var.prefix}-${var.domain}.apd.payment_option", + ] + keys = [ + { + name = "${var.prefix}-${var.domain}.apd.payment_option" + listen = true + send = true + manage = false + } + ] + }, + { + name = "${var.prefix}-${var.domain}.apd.payment_option_metadata" + partitions = 1 + message_retention = 1 + consumers = [ + "${var.prefix}-${var.domain}.apd.payment_option_metadata", + ] + keys = [ + { + name = "${var.prefix}-${var.domain}.apd.payment_option_metadata" + listen = true + send = true + manage = false + } + ] + }, + { + name = "${var.prefix}-${var.domain}.apd.payment_position" + partitions = 1 + message_retention = 1 + consumers = [ + "${var.prefix}-${var.domain}.apd.payment_position", + ] + keys = [ + { + name = "${var.prefix}-${var.domain}.apd.payment_position" + listen = true + send = true + manage = false + } + ] + }, + { + name = "${var.prefix}-${var.domain}.apd.transfer" + partitions = 1 + message_retention = 1 + consumers = [ + "${var.prefix}-${var.domain}.apd.transfer", + ] + keys = [ + { + name = "${var.prefix}-${var.domain}.apd.transfer" + listen = true + send = true + manage = false + } + ] + }, + { + name = "${var.prefix}-${var.domain}.apd.transfer_metadata" + partitions = 1 + message_retention = 1 + consumers = [ + "${var.prefix}-${var.domain}.apd.transfer_metadata", + ] + keys = [ + { + name = "${var.prefix}-${var.domain}.apd.transfer_metadata" + listen = true + send = true + manage = false + } + ] + }, + ] +} resource "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" { name = "cdc-connection-string" From b966aa5f569cd784f174da2424a82ac37913690c Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 28 Oct 2024 17:39:54 +0100 Subject: [PATCH 06/55] fix --- .../gpdingestion-common/.terraform.lock.hcl | 65 ---- src/domains/gpdingestion-common/00_data.tf | 4 - src/domains/gpdingestion-common/00_monitor.tf | 45 --- src/domains/gpdingestion-common/00_network.tf | 37 -- src/domains/gpdingestion-common/01_network.tf | 14 - .../gpdingestion-common/03_eventhub.tf | 200 ----------- .../gpdingestion-common/10_github_identity.tf | 207 ----------- src/domains/gpdingestion-common/99_locals.tf | 32 -- src/domains/gpdingestion-common/99_main.tf | 31 -- .../gpdingestion-common/99_variables.tf | 219 ------------ src/domains/gpdingestion-common/README.md | 90 ----- .../env/itn-dev/backend.ini | 1 - .../env/itn-dev/backend.tfvars | 4 - .../env/itn-dev/terraform.tfvars | 59 ---- .../env/itn-prod/backend.ini | 1 - .../env/itn-prod/backend.tfvars | 4 - .../env/itn-prod/terraform.tfvars | 58 ---- .../env/itn-uat/backend.ini | 1 - .../env/itn-uat/backend.tfvars | 4 - .../env/itn-uat/terraform.tfvars | 58 ---- src/domains/gpdingestion-common/terraform.sh | 324 ------------------ src/domains/observability/.terraform.lock.hcl | 19 + src/domains/observability/01_network.tf | 9 + .../observability/03_eventhub_msg_gdp.tf | 65 ++++ src/domains/observability/99_variables.tf | 62 +++- .../observability/env/dev/terraform.tfvars | 169 ++++++++- .../observability/env/prod/terraform.tfvars | 170 ++++++++- .../observability/env/uat/terraform.tfvars | 170 ++++++++- 28 files changed, 660 insertions(+), 1462 deletions(-) delete mode 100644 src/domains/gpdingestion-common/.terraform.lock.hcl delete mode 100644 src/domains/gpdingestion-common/00_data.tf delete mode 100644 src/domains/gpdingestion-common/00_monitor.tf delete mode 100644 src/domains/gpdingestion-common/00_network.tf delete mode 100644 src/domains/gpdingestion-common/01_network.tf delete mode 100644 src/domains/gpdingestion-common/03_eventhub.tf delete mode 100644 src/domains/gpdingestion-common/10_github_identity.tf delete mode 100644 src/domains/gpdingestion-common/99_locals.tf delete mode 100644 src/domains/gpdingestion-common/99_main.tf delete mode 100644 src/domains/gpdingestion-common/99_variables.tf delete mode 100644 src/domains/gpdingestion-common/README.md delete mode 100644 src/domains/gpdingestion-common/env/itn-dev/backend.ini delete mode 100644 src/domains/gpdingestion-common/env/itn-dev/backend.tfvars delete mode 100644 src/domains/gpdingestion-common/env/itn-dev/terraform.tfvars delete mode 100644 src/domains/gpdingestion-common/env/itn-prod/backend.ini delete mode 100644 src/domains/gpdingestion-common/env/itn-prod/backend.tfvars delete mode 100644 src/domains/gpdingestion-common/env/itn-prod/terraform.tfvars delete mode 100644 src/domains/gpdingestion-common/env/itn-uat/backend.ini delete mode 100644 src/domains/gpdingestion-common/env/itn-uat/backend.tfvars delete mode 100644 src/domains/gpdingestion-common/env/itn-uat/terraform.tfvars delete mode 100755 src/domains/gpdingestion-common/terraform.sh create mode 100644 src/domains/observability/03_eventhub_msg_gdp.tf diff --git a/src/domains/gpdingestion-common/.terraform.lock.hcl b/src/domains/gpdingestion-common/.terraform.lock.hcl deleted file mode 100644 index c1bc80ffb2..0000000000 --- a/src/domains/gpdingestion-common/.terraform.lock.hcl +++ /dev/null @@ -1,65 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/azuread" { - version = "2.47.0" - constraints = "<= 2.47.0" - hashes = [ - "h1:g8+gBFM4QVOEQFqAEs5pR6iXpbGvgPvcEi1evHwziyw=", - "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", - "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", - "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", - "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", - "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", - "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", - "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", - "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", - "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", - "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", - "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", - "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", - "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", - ] -} - -provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.106.0" - constraints = "~> 3.30, <= 3.106.0" - hashes = [ - "h1:6t9Nz9tYAR9BfHZ8yc56m+GKRl0nriwjQ5DyA0/TnCs=", - "h1:Mxe1/I27IZK3BP6cm84Gt0+7PXd2EDaDUMxuljm/rUA=", - "zh:07980d6fdc40c0adb670c8413a5c667917d6dbb51fcedc467c35d64c2f3a1f47", - "zh:2e6e8491b1f089644b0d23f8da83398f1e10cf5a62b16efcef2b5454fe923038", - "zh:450dbd72821c5619cc3bcdc20fdd0e29515147e44b733f9c79d3a75851810055", - "zh:5e234c0a2f3c9677ea72b2a6e6ca90defb99fab29ae565f5d1f70728ba4ba78f", - "zh:83fd042ece6977429d79affd03d6ce963d2f122604dbf15a1abf203d7a7bbc8a", - "zh:93027e1f66b3bf83398d572d4e6f6e7777330c78c54da3226dadd50fd868ada9", - "zh:ae3d1dd66140c303df97d93c47a60f16735ce17cf156f45475dcee4a7360af5b", - "zh:daf9d2eb89e785458a76b88bf2ef0696c472094c77cc9cff3b3ea4b885c5a482", - "zh:dd46370141651e6549da6d85e25c7a6770c47581bbaaa27eda2886d41d849747", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f77405c0d8f6e0d93d9da83256b3b02c164bad4c791ed9604310ff02ae086ad1", - "zh:ffa769147bda833aef8802e3a391bd175ec749862764d61cbdaa8200d5b8f893", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.2" - constraints = "<= 3.2.2" - hashes = [ - "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", - "h1:vWAsYRd7MjYr3adj8BVKRohVfHpWQdvkIwUQ2Jf5FVM=", - "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", - "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", - "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", - "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", - "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", - "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", - "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", - "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", - "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", - "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", - ] -} diff --git a/src/domains/gpdingestion-common/00_data.tf b/src/domains/gpdingestion-common/00_data.tf deleted file mode 100644 index 453409f78e..0000000000 --- a/src/domains/gpdingestion-common/00_data.tf +++ /dev/null @@ -1,4 +0,0 @@ -data "azurerm_key_vault" "kv" { - name = "${local.project}-kv" - resource_group_name = "${local.project}-sec-rg" -} diff --git a/src/domains/gpdingestion-common/00_monitor.tf b/src/domains/gpdingestion-common/00_monitor.tf deleted file mode 100644 index 3be0e0c27b..0000000000 --- a/src/domains/gpdingestion-common/00_monitor.tf +++ /dev/null @@ -1,45 +0,0 @@ -# -# 🇮🇹 Monitor Italy -# -data "azurerm_resource_group" "monitor_italy_rg" { - name = var.monitor_italy_resource_group_name -} - -data "azurerm_log_analytics_workspace" "log_analytics_italy" { - name = var.log_analytics_italy_workspace_name - resource_group_name = var.log_analytics_italy_workspace_resource_group_name -} - -data "azurerm_application_insights" "application_insights_italy" { - name = local.monitor_appinsights_italy_name - resource_group_name = data.azurerm_resource_group.monitor_italy_rg.name -} - -# ### 🇪🇺 -# data "azurerm_resource_group" "monitor_rg" { -# name = var.monitor_resource_group_name -# } -# -# data "azurerm_log_analytics_workspace" "log_analytics" { -# name = var.log_analytics_workspace_name -# resource_group_name = var.log_analytics_workspace_resource_group_name -# } -# -# data "azurerm_application_insights" "application_insights" { -# name = local.monitor_appinsights_name -# resource_group_name = data.azurerm_resource_group.monitor_rg.name -# } - -# -# Action Groups -# -data "azurerm_monitor_action_group" "slack" { - resource_group_name = var.monitor_resource_group_name - name = local.monitor_action_group_slack_name -} - -data "azurerm_monitor_action_group" "email" { - resource_group_name = var.monitor_resource_group_name - name = local.monitor_action_group_email_name -} - diff --git a/src/domains/gpdingestion-common/00_network.tf b/src/domains/gpdingestion-common/00_network.tf deleted file mode 100644 index 73fad2990a..0000000000 --- a/src/domains/gpdingestion-common/00_network.tf +++ /dev/null @@ -1,37 +0,0 @@ -data "azurerm_virtual_network" "vnet_italy" { - name = local.vnet_italy_name - resource_group_name = local.vnet_italy_resource_group_name -} - -data "azurerm_resource_group" "rg_vnet_italy" { - name = local.vnet_italy_resource_group_name -} - -# -# Subnets -# -data "azurerm_subnet" "aks_subnet" { - name = local.aks_subnet_name - virtual_network_name = local.vnet_italy_name - resource_group_name = local.vnet_italy_resource_group_name -} - -# -# Private DNS Zones -# -data "azurerm_private_dns_zone" "internal" { - name = local.internal_dns_zone_name - resource_group_name = local.internal_dns_zone_resource_group_name -} - -# -# Eventhub -# -data "azurerm_private_dns_zone" "eventhub" { - name = "privatelink.servicebus.windows.net" - resource_group_name = local.msg_resource_group_name -} - -data "azurerm_resource_group" "rg_event_private_dns_zone" { - name = local.msg_resource_group_name -} diff --git a/src/domains/gpdingestion-common/01_network.tf b/src/domains/gpdingestion-common/01_network.tf deleted file mode 100644 index 7a80e7444b..0000000000 --- a/src/domains/gpdingestion-common/01_network.tf +++ /dev/null @@ -1,14 +0,0 @@ -resource "azurerm_private_dns_a_record" "ingress" { - name = local.ingress_hostname - zone_name = data.azurerm_private_dns_zone.internal.name - resource_group_name = local.internal_dns_zone_resource_group_name - ttl = 3600 - records = [var.ingress_load_balancer_ip] -} - -resource "azurerm_subnet" "eventhub_italy" { - name = "${local.project}-eventhub-snet" - resource_group_name = data.azurerm_resource_group.rg_vnet_italy.name - virtual_network_name = data.azurerm_virtual_network.vnet_italy.name - address_prefixes = var.cidr_gpdingestion_eventhub_italy -} diff --git a/src/domains/gpdingestion-common/03_eventhub.tf b/src/domains/gpdingestion-common/03_eventhub.tf deleted file mode 100644 index 0592919026..0000000000 --- a/src/domains/gpdingestion-common/03_eventhub.tf +++ /dev/null @@ -1,200 +0,0 @@ -resource "azurerm_resource_group" "eventhub_ita_rg" { - name = local.eventhub_resource_group_name - location = var.location - - tags = var.tags -} - -module "eventhub_namespace" { - source = "./.terraform/modules/__v3__/eventhub_configuration" - name = "${local.project}-evh" - location = var.location - resource_group_name = azurerm_resource_group.eventhub_ita_rg.name - auto_inflate_enabled = var.ehns_auto_inflate_enabled - sku = var.ehns_sku_name - capacity = var.ehns_capacity - maximum_throughput_units = var.ehns_maximum_throughput_units - #zone_redundat is always true - - virtual_network_ids = [data.azurerm_virtual_network.vnet_italy.id] - private_endpoint_subnet_id = azurerm_subnet.eventhub_italy.id - public_network_access_enabled = var.ehns_public_network_access - private_endpoint_created = var.ehns_private_endpoint_is_present - - private_endpoint_resource_group_name = azurerm_resource_group.eventhub_ita_rg.name - - private_dns_zones = { - id = [data.azurerm_private_dns_zone.eventhub.id] - name = [data.azurerm_private_dns_zone.eventhub.name] - resource_group_name = data.azurerm_resource_group.rg_event_private_dns_zone.name - } - - private_dns_zone_record_A_name = "${var.domain}.${var.location_short}" - - action = [ - { - action_group_id = data.azurerm_monitor_action_group.slack.id - webhook_properties = null - }, - { - action_group_id = data.azurerm_monitor_action_group.email.id - webhook_properties = null - } - ] - - metric_alerts_create = var.ehns_alerts_enabled - metric_alerts = var.ehns_metric_alerts - - tags = var.tags -} - - - CONFIGURATION - -module "eventhub_gpdingestion_configuration" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration?ref=v8.22.0" - count = var.is_feature_enabled.eventhub ? 1 : 0 - - event_hub_namespace_name = module.eventhub_namespace.name - event_hub_namespace_resource_group_name = azurerm_resource_group.eventhub_ita_rg.name - - eventhubs = [ - { - name = "connect-cluster-offsets" - partitions = 1 - message_retention = 1 - consumers = [ - "connect-cluster-offsets", - ] - keys = [ - { - name = "connect-cluster-offsets" - listen = true - send = true - manage = false - } - ] - }, - { - name = "connect-cluster-status" - partitions = 1 - message_retention = 1 - consumers = [ - "connect-cluster-offsets", - ] - keys = [ - { - name = "connect-cluster-status" - listen = true - send = true - manage = false - } - ] - }, - { - name = "connect-cluster-configs" - partitions = 1 - message_retention = 1 - consumers = [ - "connect-cluster-configs", - ] - keys = [ - { - name = "connect-cluster-configs" - listen = true - send = true - manage = false - } - ] - }, - { - name = "${var.prefix}-${var.domain}.apd.payment_option" - partitions = 1 - message_retention = 1 - consumers = [ - "${var.prefix}-${var.domain}.apd.payment_option", - ] - keys = [ - { - name = "${var.prefix}-${var.domain}.apd.payment_option" - listen = true - send = true - manage = false - } - ] - }, - { - name = "${var.prefix}-${var.domain}.apd.payment_option_metadata" - partitions = 1 - message_retention = 1 - consumers = [ - "${var.prefix}-${var.domain}.apd.payment_option_metadata", - ] - keys = [ - { - name = "${var.prefix}-${var.domain}.apd.payment_option_metadata" - listen = true - send = true - manage = false - } - ] - }, - { - name = "${var.prefix}-${var.domain}.apd.payment_position" - partitions = 1 - message_retention = 1 - consumers = [ - "${var.prefix}-${var.domain}.apd.payment_position", - ] - keys = [ - { - name = "${var.prefix}-${var.domain}.apd.payment_position" - listen = true - send = true - manage = false - } - ] - }, - { - name = "${var.prefix}-${var.domain}.apd.transfer" - partitions = 1 - message_retention = 1 - consumers = [ - "${var.prefix}-${var.domain}.apd.transfer", - ] - keys = [ - { - name = "${var.prefix}-${var.domain}.apd.transfer" - listen = true - send = true - manage = false - } - ] - }, - { - name = "${var.prefix}-${var.domain}.apd.transfer_metadata" - partitions = 1 - message_retention = 1 - consumers = [ - "${var.prefix}-${var.domain}.apd.transfer_metadata", - ] - keys = [ - { - name = "${var.prefix}-${var.domain}.apd.transfer_metadata" - listen = true - send = true - manage = false - } - ] - }, - ] -} - -resource "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" { - name = "cdc-connection-string" - namespace_name = "${local.project}-evh" - resource_group_name = "${local.project}-evh-rg" - listen = true - send = true - manage = true -} diff --git a/src/domains/gpdingestion-common/10_github_identity.tf b/src/domains/gpdingestion-common/10_github_identity.tf deleted file mode 100644 index d59a9d9cbe..0000000000 --- a/src/domains/gpdingestion-common/10_github_identity.tf +++ /dev/null @@ -1,207 +0,0 @@ -data "azurerm_resource_group" "identity_rg" { - name = "${local.product}-identity-rg" -} - -data "azurerm_kubernetes_cluster" "aks" { - name = "${local.product}-${var.location_short}-${var.instance}-aks" - resource_group_name = "${local.product}-${var.location_short}-${var.instance}-aks-rg" -} - -data "azurerm_key_vault" "key_vault" { - name = "${local.product}-${var.location_short}-${var.domain}-kv" - resource_group_name = "${local.product}-${var.location_short}-${var.domain}-sec-rg" -} - -# repos must be lower than 20 items -locals { - repos_01 = [ - "pagopa-gpd-ingestion-manager", - ] - - federations_01 = [ - for repo in local.repos_01 : { - repository = repo - subject = var.env - } - ] - - federations_01_pr = [ - for repo in local.repos_01 : { - repository = repo - subject = "pull_request" - } - ] - - federations_01_ref = [ - for repo in local.repos_01 : { - repository = repo - credentials_scope = "ref" - subject = "refs/heads/main" - } - ] - - - # to avoid subscription Contributor -> https://github.com/microsoft/azure-container-apps/issues/35 - environment_cd_roles = { - subscription = [ - "Contributor", - ] - resource_groups = { - "${local.product}-${var.location_short}-${var.domain}-sec-rg" = [ - "Key Vault Reader" - ], - "${local.product}-${var.location_short}-${var.env}-aks-rg" = [ - "Contributor" - ], - } - } -} - -# create a module for each 20 repos -module "identity_cd_01" { - source = "./.terraform/modules/__v3__/github_federated_identity" - # pagopa---github--identity - prefix = var.prefix - env_short = var.env_short - domain = "${var.domain}-01" - - identity_role = "cd" - - github_federations = local.federations_01 - - cd_rbac_roles = { - subscription_roles = local.environment_cd_roles.subscription - resource_groups = local.environment_cd_roles.resource_groups - } - - tags = var.tags - - depends_on = [ - data.azurerm_resource_group.identity_rg - ] -} - -resource "azurerm_key_vault_access_policy" "gha_iac_managed_identities" { - key_vault_id = data.azurerm_key_vault.key_vault.id - tenant_id = data.azurerm_client_config.current.tenant_id - object_id = module.identity_cd_01.identity_principal_id - - secret_permissions = ["Get", "List", "Set", ] - - certificate_permissions = ["SetIssuers", "DeleteIssuers", "Purge", "List", "Get"] - key_permissions = [ - "Get", "List", "Update", "Create", "Import", "Delete", "Encrypt", "Decrypt", "GetRotationPolicy" - ] - - storage_permissions = [] -} - -resource "null_resource" "github_runner_app_permissions_to_namespace_cd_01" { - triggers = { - aks_id = data.azurerm_kubernetes_cluster.aks.id - service_principal_id = module.identity_cd_01.identity_client_id - namespace = var.domain - version = "v2" - } - - provisioner "local-exec" { - command = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.6 | -| [azuread](#requirement\_azuread) | <= 2.47.0 | -| [azurerm](#requirement\_azurerm) | <= 3.106.0 | -| [null](#requirement\_null) | <= 3.2.2 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [eventhub\_namespace](#module\_eventhub\_namespace) | git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub | v8.22.0 | -| [eventhub\_paymentoptions\_configuration](#module\_eventhub\_paymentoptions\_configuration) | git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration | v8.22.0 | -| [identity\_cd\_01](#module\_identity\_cd\_01) | github.com/pagopa/terraform-azurerm-v3//github_federated_identity | v8.22.0 | -| [identity\_pr\_01](#module\_identity\_pr\_01) | github.com/pagopa/terraform-azurerm-v3//github_federated_identity | v8.22.0 | -| [identity\_ref\_01](#module\_identity\_ref\_01) | github.com/pagopa/terraform-azurerm-v3//github_federated_identity | v8.36.1 | - -## Resources - -| Name | Type | -|------|------| -| [azurerm_key_vault_access_policy.gha_iac_managed_identities](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | -| [azurerm_key_vault_access_policy.gha_pr_iac_managed_identities](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | -| [azurerm_key_vault_access_policy.gha_ref_iac_managed_identities](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | -| [azurerm_private_dns_a_record.ingress](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_a_record) | resource | -| [azurerm_resource_group.eventhub_ita_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | -| [azurerm_subnet.eventhub_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | -| [null_resource.github_runner_app_permissions_to_namespace_cd_01](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [azurerm_application_insights.application_insights_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/application_insights) | data source | -| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | -| [azurerm_key_vault.key_vault](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault) | data source | -| [azurerm_key_vault.kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault) | data source | -| [azurerm_kubernetes_cluster.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_cluster) | data source | -| [azurerm_log_analytics_workspace.log_analytics_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/log_analytics_workspace) | data source | -| [azurerm_monitor_action_group.email](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | -| [azurerm_monitor_action_group.slack](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | -| [azurerm_private_dns_zone.eventhub](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source | -| [azurerm_private_dns_zone.internal](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source | -| [azurerm_resource_group.identity_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | -| [azurerm_resource_group.monitor_italy_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | -| [azurerm_resource_group.rg_event_private_dns_zone](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | -| [azurerm_resource_group.rg_vnet_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | -| [azurerm_subnet.aks_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | -| [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | -| [azurerm_virtual_network.vnet_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/virtual_network) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [cidr\_paymentoptions\_eventhub\_italy](#input\_cidr\_paymentoptions\_eventhub\_italy) | Address prefixes for all evh accounts in italy. | `list(string)` | n/a | yes | -| [dns\_zone\_internal\_prefix](#input\_dns\_zone\_internal\_prefix) | The dns subdomain. | `string` | `null` | no | -| [dns\_zone\_platform](#input\_dns\_zone\_platform) | The platform dns subdomain. | `string` | `null` | no | -| [dns\_zone\_prefix](#input\_dns\_zone\_prefix) | The wallet dns subdomain. | `string` | `null` | no | -| [domain](#input\_domain) | n/a | `string` | n/a | yes | -| [ehns\_alerts\_enabled](#input\_ehns\_alerts\_enabled) | Event hub alerts enabled? | `bool` | n/a | yes | -| [ehns\_auto\_inflate\_enabled](#input\_ehns\_auto\_inflate\_enabled) | Is Auto Inflate enabled for the EventHub Namespace? | `bool` | n/a | yes | -| [ehns\_capacity](#input\_ehns\_capacity) | Specifies the Capacity / Throughput Units for a Standard SKU namespace. | `number` | n/a | yes | -| [ehns\_maximum\_throughput\_units](#input\_ehns\_maximum\_throughput\_units) | Specifies the maximum number of throughput units when Auto Inflate is Enabled | `number` | n/a | yes | -| [ehns\_metric\_alerts](#input\_ehns\_metric\_alerts) | Map of name = criteria objects | 
map(object({
    # criteria.*.aggregation to be one of [Average Count Minimum Maximum Total]
    aggregation = string
    metric_name = string
    description = string
    # criteria.0.operator to be one of [Equals NotEquals GreaterThan GreaterThanOrEqual LessThan LessThanOrEqual]
    operator  = string
    threshold = number
    # Possible values are PT1M, PT5M, PT15M, PT30M and PT1H
    frequency = string
    # Possible values are PT1M, PT5M, PT15M, PT30M, PT1H, PT6H, PT12H and P1D.
    window_size = string

    dimension = list(object(
      {
        name     = string
        operator = string
        values   = list(string)
      }
    ))
  }))
| `{}` | no | -| [ehns\_private\_endpoint\_is\_present](#input\_ehns\_private\_endpoint\_is\_present) | (Required) create private endpoint to the event hubs | `bool` | n/a | yes | -| [ehns\_public\_network\_access](#input\_ehns\_public\_network\_access) | (Required) enables public network access to the event hubs | `bool` | n/a | yes | -| [ehns\_sku\_name](#input\_ehns\_sku\_name) | Defines which tier to use. | `string` | n/a | yes | -| [ehns\_zone\_redundant](#input\_ehns\_zone\_redundant) | Specifies if the EventHub Namespace should be Zone Redundant (created across Availability Zones). | `bool` | n/a | yes | -| [env](#input\_env) | n/a | `string` | n/a | yes | -| [env\_short](#input\_env\_short) | n/a | `string` | n/a | yes | -| [external\_domain](#input\_external\_domain) | Domain for delegation | `string` | `null` | no | -| [ingress\_load\_balancer\_ip](#input\_ingress\_load\_balancer\_ip) | n/a | `string` | n/a | yes | -| [instance](#input\_instance) | One of beta, prod01, prod02 | `string` | n/a | yes | -| [is\_feature\_enabled](#input\_is\_feature\_enabled) | n/a | 
object({
    eventhub = bool
  })
| 
{
  "eventhub": false
}
| no | -| [location](#input\_location) | One of westeurope, northeurope | `string` | n/a | yes | -| [location\_short](#input\_location\_short) | One of wue, neu | `string` | `"itn"` | no | -| [log\_analytics\_italy\_workspace\_name](#input\_log\_analytics\_italy\_workspace\_name) | Specifies the name of the Log Analytics Workspace Italy. | `string` | n/a | yes | -| [log\_analytics\_italy\_workspace\_resource\_group\_name](#input\_log\_analytics\_italy\_workspace\_resource\_group\_name) | The name of the resource group in which the Log Analytics workspace Italy is located in. | `string` | n/a | yes | -| [log\_analytics\_workspace\_name](#input\_log\_analytics\_workspace\_name) | Specifies the name of the Log Analytics Workspace. | `string` | n/a | yes | -| [log\_analytics\_workspace\_resource\_group\_name](#input\_log\_analytics\_workspace\_resource\_group\_name) | The name of the resource group in which the Log Analytics workspace is located in. | `string` | n/a | yes | -| [monitor\_italy\_resource\_group\_name](#input\_monitor\_italy\_resource\_group\_name) | Monitor Italy resource group name | `string` | n/a | yes | -| [monitor\_resource\_group\_name](#input\_monitor\_resource\_group\_name) | Monitor resource group name | `string` | n/a | yes | -| [prefix](#input\_prefix) | general | `string` | n/a | yes | -| [tags](#input\_tags) | n/a | `map(any)` | 
{
  "CreatedBy": "Terraform"
}
| no | - -## Outputs - -No outputs. - diff --git a/src/domains/gpdingestion-common/env/itn-dev/backend.ini b/src/domains/gpdingestion-common/env/itn-dev/backend.ini deleted file mode 100644 index f3ea2d530c..0000000000 --- a/src/domains/gpdingestion-common/env/itn-dev/backend.ini +++ /dev/null @@ -1 +0,0 @@ -subscription=DEV-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-common/env/itn-dev/backend.tfvars b/src/domains/gpdingestion-common/env/itn-dev/backend.tfvars deleted file mode 100644 index 61e0c7b275..0000000000 --- a/src/domains/gpdingestion-common/env/itn-dev/backend.tfvars +++ /dev/null @@ -1,4 +0,0 @@ -resource_group_name = "terraform-state-rg" -storage_account_name = "tfinfdevpagopa" -container_name = "terraform-state" -key = "gpdingestion-common-dev.terraform.tfstate" diff --git a/src/domains/gpdingestion-common/env/itn-dev/terraform.tfvars b/src/domains/gpdingestion-common/env/itn-dev/terraform.tfvars deleted file mode 100644 index f2714d8780..0000000000 --- a/src/domains/gpdingestion-common/env/itn-dev/terraform.tfvars +++ /dev/null @@ -1,59 +0,0 @@ -prefix = "pagopa" -env_short = "d" -env = "dev" -domain = "gpdingestion" -location = "italynorth" -location_short = "itn" -instance = "dev" - -tags = { - CreatedBy = "Terraform" - Environment = "Dev" - Owner = "pagoPA" - Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-common" - CostCenter = "TS310 - PAGAMENTI & SERVIZI" -} - -### 🚩Features flags - -is_feature_enabled = { - eventhub = true -} - -### CIRDs - -cidr_gpdingestion_eventhub_italy = ["10.3.13.0/27"] - -### External resources - -monitor_italy_resource_group_name = "pagopa-d-itn-core-monitor-rg" -log_analytics_italy_workspace_name = "pagopa-d-itn-core-law" -log_analytics_italy_workspace_resource_group_name = "pagopa-d-itn-core-monitor-rg" - -monitor_resource_group_name = "pagopa-d-monitor-rg" -log_analytics_workspace_name = "pagopa-d-law" -log_analytics_workspace_resource_group_name = "pagopa-d-monitor-rg" - -### Aks - -ingress_load_balancer_ip = "10.3.100.250" - -external_domain = "pagopa.it" -dns_zone_internal_prefix = "internal.dev.platform" - -# -# EventHub -# -ehns_sku_name = "Standard" - -# to avoid https://docs.microsoft.com/it-it/azure/event-hubs/event-hubs-messaging-exceptions#error-code-50002 -ehns_auto_inflate_enabled = false -ehns_maximum_throughput_units = 5 -ehns_capacity = 1 -ehns_alerts_enabled = false -ehns_zone_redundant = false - -ehns_public_network_access = true -ehns_private_endpoint_is_present = false - - diff --git a/src/domains/gpdingestion-common/env/itn-prod/backend.ini b/src/domains/gpdingestion-common/env/itn-prod/backend.ini deleted file mode 100644 index 432abea37c..0000000000 --- a/src/domains/gpdingestion-common/env/itn-prod/backend.ini +++ /dev/null @@ -1 +0,0 @@ -subscription=PROD-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-common/env/itn-prod/backend.tfvars b/src/domains/gpdingestion-common/env/itn-prod/backend.tfvars deleted file mode 100644 index 6146f86e2c..0000000000 --- a/src/domains/gpdingestion-common/env/itn-prod/backend.tfvars +++ /dev/null @@ -1,4 +0,0 @@ -resource_group_name = "terraform-state-rg" -storage_account_name = "tfinfprodpagopa" -container_name = "terraform-state" -key = "gpdingestion-common-prod.terraform.tfstate" diff --git a/src/domains/gpdingestion-common/env/itn-prod/terraform.tfvars b/src/domains/gpdingestion-common/env/itn-prod/terraform.tfvars deleted file mode 100644 index 47e0b7993f..0000000000 --- a/src/domains/gpdingestion-common/env/itn-prod/terraform.tfvars +++ /dev/null @@ -1,58 +0,0 @@ -prefix = "pagopa" -env_short = "p" -env = "prod" -domain = "payopt" -location = "italynorth" -location_short = "itn" -instance = "prod" - -tags = { - CreatedBy = "Terraform" - Environment = "Prod" - Owner = "pagoPA" - Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-common" - CostCenter = "TS310 - PAGAMENTI & SERVIZI" -} - -### 🚩Features flags - -is_feature_enabled = { - eventhub = true -} - -### CIRDs - -cidr_gpdingestion_eventhub_italy = ["10.3.13.0/27"] - -### External resources - -monitor_italy_resource_group_name = "pagopa-p-itn-core-monitor-rg" -log_analytics_italy_workspace_name = "pagopa-p-itn-core-law" -log_analytics_italy_workspace_resource_group_name = "pagopa-p-itn-core-monitor-rg" - -monitor_resource_group_name = "pagopa-p-monitor-rg" -log_analytics_workspace_name = "pagopa-p-law" -log_analytics_workspace_resource_group_name = "pagopa-p-monitor-rg" - -### Aks - -ingress_load_balancer_ip = "10.3.100.250" - -external_domain = "pagopa.it" -dns_zone_internal_prefix = "internal.platform" - -# -# EventHub -# -ehns_sku_name = "Standard" - -# to avoid https://docs.microsoft.com/it-it/azure/event-hubs/event-hubs-messaging-exceptions#error-code-50002 -ehns_auto_inflate_enabled = true -ehns_maximum_throughput_units = 5 -ehns_capacity = 5 -ehns_alerts_enabled = true -ehns_zone_redundant = true - -ehns_public_network_access = false -ehns_private_endpoint_is_present = true - diff --git a/src/domains/gpdingestion-common/env/itn-uat/backend.ini b/src/domains/gpdingestion-common/env/itn-uat/backend.ini deleted file mode 100644 index 1759a0ca0d..0000000000 --- a/src/domains/gpdingestion-common/env/itn-uat/backend.ini +++ /dev/null @@ -1 +0,0 @@ -subscription=UAT-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-common/env/itn-uat/backend.tfvars b/src/domains/gpdingestion-common/env/itn-uat/backend.tfvars deleted file mode 100644 index 3eca13f707..0000000000 --- a/src/domains/gpdingestion-common/env/itn-uat/backend.tfvars +++ /dev/null @@ -1,4 +0,0 @@ -resource_group_name = "terraform-state-rg" -storage_account_name = "tfinfuatpagopa" -container_name = "terraform-state" -key = "gpdingestion-common-uat.terraform.tfstate" diff --git a/src/domains/gpdingestion-common/env/itn-uat/terraform.tfvars b/src/domains/gpdingestion-common/env/itn-uat/terraform.tfvars deleted file mode 100644 index 019de1bf08..0000000000 --- a/src/domains/gpdingestion-common/env/itn-uat/terraform.tfvars +++ /dev/null @@ -1,58 +0,0 @@ -prefix = "pagopa" -env_short = "u" -env = "uat" -domain = "gpdingestion" -location = "italynorth" -location_short = "itn" -instance = "uat" - -tags = { - CreatedBy = "Terraform" - Environment = "Uat" - Owner = "pagoPA" - Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-common" - CostCenter = "TS310 - PAGAMENTI & SERVIZI" -} - -### 🚩Features flags - -is_feature_enabled = { - eventhub = true -} - -### CIRDs - -cidr_gpdingestion_eventhub_italy = ["10.3.13.0/27"] - -### External resources - -monitor_italy_resource_group_name = "pagopa-u-itn-core-monitor-rg" -log_analytics_italy_workspace_name = "pagopa-u-itn-core-law" -log_analytics_italy_workspace_resource_group_name = "pagopa-u-itn-core-monitor-rg" - -monitor_resource_group_name = "pagopa-u-monitor-rg" -log_analytics_workspace_name = "pagopa-u-law" -log_analytics_workspace_resource_group_name = "pagopa-u-monitor-rg" - -### Aks - -ingress_load_balancer_ip = "10.3.100.250" - -external_domain = "pagopa.it" -dns_zone_internal_prefix = "internal.uat.platform" - -# -# EventHub -# -ehns_sku_name = "Standard" - -# to avoid https://docs.microsoft.com/it-it/azure/event-hubs/event-hubs-messaging-exceptions#error-code-50002 -ehns_auto_inflate_enabled = true -ehns_maximum_throughput_units = 5 -ehns_capacity = 1 -ehns_alerts_enabled = false -ehns_zone_redundant = false - -ehns_public_network_access = false -ehns_private_endpoint_is_present = true - diff --git a/src/domains/gpdingestion-common/terraform.sh b/src/domains/gpdingestion-common/terraform.sh deleted file mode 100755 index 047a7512d0..0000000000 --- a/src/domains/gpdingestion-common/terraform.sh +++ /dev/null @@ -1,324 +0,0 @@ -#!/bin/bash -############################################################ -# Terraform script for managing infrastructure on Azure -# Fingerprint: d2hhdHlvdXdhbnQ/Cg== -############################################################ -# Global variables -# Version format x.y accepted -vers="1.11" -script_name=$(basename "$0") -git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/azure/${script_name}" -tmp_file="${script_name}.new" -# Check if the third parameter exists and is a file -if [ -n "$3" ] && [ -f "$3" ]; then - FILE_ACTION=true -else - FILE_ACTION=false -fi - -# Define functions -function clean_environment() { - rm -rf .terraform - rm tfplan 2>/dev/null - echo "cleaned!" -} - -function download_tool() { - #default value - cpu_type="intel" - os_type=$(uname) - - # only on MacOS - if [ "$os_type" == "Darwin" ]; then - cpu_brand=$(sysctl -n machdep.cpu.brand_string) - if grep -q -i "intel" <<< "$cpu_brand"; then - cpu_type="intel" - else - cpu_type="arm" - fi - fi - - echo $cpu_type - tool=$1 - git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/golang/${tool}_${cpu_type}" - if ! command -v $tool &> /dev/null; then - if ! curl -sL "$git_repo" -o "$tool"; then - echo "Error downloading ${tool}" - return 1 - else - chmod +x $tool - echo "${tool} downloaded! Please note this tool WON'T be copied in your **/bin folder for safety reasons. -You need to do it yourself!" - read -p "Press enter to continue" - - - fi - fi -} - -function extract_resources() { - TF_FILE=$1 - ENV=$2 - TARGETS="" - - # Check if the file exists - if [ ! -f "$TF_FILE" ]; then - echo "File $TF_FILE does not exist." - exit 1 - fi - - # Check if the directory exists - if [ ! -d "./env/$ENV" ]; then - echo "Directory ./env/$ENV does not exist." - exit 1 - fi - - TMP_FILE=$(mktemp) - grep -E '^resource|^module' $TF_FILE > $TMP_FILE - - while read -r line ; do - TYPE=$(echo $line | cut -d '"' -f 1 | tr -d ' ') - if [ "$TYPE" == "module" ]; then - NAME=$(echo $line | cut -d '"' -f 2) - TARGETS+=" -target=\"$TYPE.$NAME\"" - else - NAME1=$(echo $line | cut -d '"' -f 2) - NAME2=$(echo $line | cut -d '"' -f 4) - TARGETS+=" -target=\"$NAME1.$NAME2\"" - fi - done < $TMP_FILE - - rm $TMP_FILE - - echo "./terraform.sh $action $ENV $TARGETS" -} - -function help_usage() { - echo "terraform.sh Version ${vers}" - echo - echo "Usage: ./script.sh [ACTION] [ENV] [OTHER OPTIONS]" - echo "es. ACTION: init, apply, plan, etc." - echo "es. ENV: dev, uat, prod, etc." - echo - echo "Available actions:" - echo " clean Remove .terraform* folders and tfplan files" - echo " help This help" - echo " list List every environment available" - echo " update Update this script if possible" - echo " summ Generate summary of Terraform plan" - echo " tflist Generate an improved output of terraform state list" - echo " tlock Generate or update the dependency lock file" - echo " * any terraform option" -} - -function init_terraform() { - if [ -n "$env" ]; then - terraform init -reconfigure -backend-config="./env/$env/backend.tfvars" - else - echo "ERROR: no env configured!" - exit 1 - fi -} - -function list_env() { - # Check if env directory exists - if [ ! -d "./env" ]; then - echo "No environment directory found" - exit 1 - fi - - # List subdirectories under env directory - env_list=$(ls -d ./env/*/ 2>/dev/null) - - # Check if there are any subdirectories - if [ -z "$env_list" ]; then - echo "No environments found" - exit 1 - fi - - # Print the list of environments - echo "Available environments:" - for env in $env_list; do - env_name=$(echo "$env" | sed 's#./env/##;s#/##') - echo "- $env_name" - done -} - -function other_actions() { - if [ -n "$env" ] && [ -n "$action" ]; then - terraform "$action" -var-file="./env/$env/terraform.tfvars" -compact-warnings $other - else - echo "ERROR: no env or action configured!" - exit 1 - fi -} - -function state_output_taint_actions() { - if [ "$action" == "tflist" ]; then - # If 'tflist' is not installed globally and there is no 'tflist' file in the current directory, - # attempt to download the 'tflist' tool - if ! command -v tflist &> /dev/null && [ ! -f "tflist" ]; then - download_tool "tflist" - if [ $? -ne 0 ]; then - echo "Error: Failed to download tflist!!" - exit 1 - else - echo "tflist downloaded!" - fi - fi - if command -v tflist &> /dev/null; then - terraform state list | tflist - else - terraform state list | ./tflist - fi - else - terraform $action $other - fi -} - - -function parse_tfplan_option() { - # Create an array to contain arguments that do not start with '-tfplan=' - local other_args=() - - # Loop over all arguments - for arg in "$@"; do - # If the argument starts with '-tfplan=', extract the file name - if [[ "$arg" =~ ^-tfplan= ]]; then - echo "${arg#*=}" - else - # If the argument does not start with '-tfplan=', add it to the other_args array - other_args+=("$arg") - fi - done - - # Print all arguments in other_args separated by spaces - echo "${other_args[@]}" -} - -function tfsummary() { - local plan_file - plan_file=$(parse_tfplan_option "$@") - if [ -z "$plan_file" ]; then - plan_file="tfplan" - fi - action="plan" - other="-out=${plan_file}" - other_actions - if [ -n "$(command -v tf-summarize)" ]; then - tf-summarize -tree "${plan_file}" - else - echo "tf-summarize is not installed" - fi - if [ "$plan_file" == "tfplan" ]; then - rm $plan_file - fi -} - -function update_script() { - # Check if the repository was cloned successfully - if ! curl -sL "$git_repo" -o "$tmp_file"; then - echo "Error cloning the repository" - rm "$tmp_file" 2>/dev/null - return 1 - fi - - # Check if a newer version exists - remote_vers=$(sed -n '8s/vers="\(.*\)"/\1/p' "$tmp_file") - if [ "$(printf '%s\n' "$vers" "$remote_vers" | sort -V | tail -n 1)" == "$vers" ]; then - echo "The local script version is equal to or newer than the remote version." - rm "$tmp_file" 2>/dev/null - return 0 - fi - - # Check the fingerprint - local_fingerprint=$(sed -n '4p' "$0") - remote_fingerprint=$(sed -n '4p' "$tmp_file") - - if [ "$local_fingerprint" != "$remote_fingerprint" ]; then - echo "The local and remote file fingerprints do not match." - rm "$tmp_file" 2>/dev/null - return 0 - fi - - # Show the current and available versions to the user - echo "Current script version: $vers" - echo "Available script version: $remote_vers" - - # Ask the user if they want to update the script - read -rp "Do you want to update the script to version $remote_vers? (y/n): " answer - - if [ "$answer" == "y" ] || [ "$answer" == "Y" ]; then - # Replace the local script with the updated version - cp "$tmp_file" "$script_name" - chmod +x "$script_name" - rm "$tmp_file" 2>/dev/null - - echo "Script successfully updated to version $remote_vers" - else - echo "Update canceled by the user" - fi - - rm "$tmp_file" 2>/dev/null -} - -# Check arguments number -if [ "$#" -lt 1 ]; then - help_usage - exit 0 -fi - -# Parse arguments -action=$1 -env=$2 -filetf=$3 -shift 2 -other=$@ - -if [ -n "$env" ]; then - # shellcheck source=/dev/null - source "./env/$env/backend.ini" - if [ -z "$(command -v az)" ]; then - echo "az not found, cannot proceed" - exit 1 - fi - az account set -s "${subscription}" -fi - -# Call appropriate function based on action -case $action in - clean) - clean_environment - ;; - ?|help|-h) - help_usage - ;; - init) - init_terraform "$other" - ;; - list) - list_env - ;; - output|state|taint|tflist) - init_terraform - state_output_taint_actions $other - ;; - summ) - init_terraform - tfsummary "$other" - ;; - tlock) - terraform providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=darwin_arm64 -platform=linux_amd64 - ;; - update) - update_script - ;; - *) - if [ "$FILE_ACTION" = true ]; then - extract_resources "$filetf" "$env" - else - init_terraform - other_actions "$other" - fi - ;; -esac diff --git a/src/domains/observability/.terraform.lock.hcl b/src/domains/observability/.terraform.lock.hcl index f8832a4a69..3078d2e4fa 100644 --- a/src/domains/observability/.terraform.lock.hcl +++ b/src/domains/observability/.terraform.lock.hcl @@ -1,6 +1,25 @@ # This file is maintained automatically by "terraform init". # Manual edits may be lost in future updates. +provider "registry.terraform.io/azure/azapi" { + version = "2.0.1" + hashes = [ + "h1:NybiPNJ4MsKsTtRUnxV5vrhkbPkK+puqgfa0j0Q5nlU=", + "zh:3df16ed604be5f4ccd5d52a02c2681d8eb2f5a4462625c983cb17c20cdf0bfb2", + "zh:4efd9961ea52990e21385086f0b3324edfb534ea6a8f0f6ba146a74bfb56aa63", + "zh:5561418efc9744c9873855a146226608778e29b4c0c3b3872634ef2da2d86593", + "zh:7ebcb4c6ca71c87850df67d4e5f79ce4a036d4131b8c11ae0b9b8787353843b8", + "zh:81a9259cb1e45507e9431794fbd354dd4d8b78c6a9508b0bfa108b00e6ad23cb", + "zh:8c1836fa186272347f97c7a3884556979618d1b93721e8a24203d90ff4efbd40", + "zh:a72bdd43a11a383525764720d24cb78ec5d9f1167f129d05448108fef1ba7af3", + "zh:ade9d17c6b8717e7b04af5a9d1a948d047ac4dcf6affb2485afa3ad0a2eaee15", + "zh:b3c5bfcab98251cb0c157dbe78dc6d0864c9bf364d316003c84c1e624a3c3524", + "zh:c33b872a2473a9b052add89e4557d361b0ebaa42865e99b95465050d2c858d43", + "zh:efe425f8ecd4d79448214c93ef10881b3b74cf2d9b5211d76f05aced22621eb4", + "zh:ff704c5e73e832507367d9d962b6b53c0ca3c724689f0974feffd5339c3db18a", + ] +} + provider "registry.terraform.io/hashicorp/azuread" { version = "2.21.0" constraints = "2.21.0" diff --git a/src/domains/observability/01_network.tf b/src/domains/observability/01_network.tf index 5c0b2c0393..9a04956941 100644 --- a/src/domains/observability/01_network.tf +++ b/src/domains/observability/01_network.tf @@ -29,9 +29,18 @@ data "azurerm_resource_group" "rg_event_private_dns_zone" { name = local.msg_resource_group_name } +# all snet for each evh(s) + resource "azurerm_subnet" "eventhub_observability_snet" { name = "${local.project_itn}-evh-observability-snet" resource_group_name = data.azurerm_resource_group.rg_vnet_italy.name virtual_network_name = data.azurerm_virtual_network.vnet_italy.name address_prefixes = var.cidr_subnet_observability_evh } + +resource "azurerm_subnet" "eventhub_observability_gpd_snet" { + name = "${local.project_itn}-evh-observability-gpd--snet" + resource_group_name = data.azurerm_resource_group.rg_vnet_italy.name + virtual_network_name = data.azurerm_virtual_network.vnet_italy.name + address_prefixes = var.cidr_subnet_observability_gpd_evh +} diff --git a/src/domains/observability/03_eventhub_msg_gdp.tf b/src/domains/observability/03_eventhub_msg_gdp.tf new file mode 100644 index 0000000000..6a4e041ae8 --- /dev/null +++ b/src/domains/observability/03_eventhub_msg_gdp.tf @@ -0,0 +1,65 @@ + +module "eventhub_namespace_observability_gpd" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub?ref=v8.22.0" + name = "${local.project_itn}-gpd-evh" + location = var.location_itn + resource_group_name = azurerm_resource_group.eventhub_observability_rg.name + auto_inflate_enabled = var.ehns_auto_inflate_enabled + sku = var.ehns_sku_name + capacity = var.ehns_capacity + maximum_throughput_units = var.ehns_maximum_throughput_units + #zone_redundat is always true + + virtual_network_ids = [data.azurerm_virtual_network.vnet_italy.id] + private_endpoint_subnet_id = azurerm_subnet.eventhub_observability_gpd_snet.id + public_network_access_enabled = var.ehns_public_network_access + private_endpoint_created = var.ehns_private_endpoint_is_present + + private_endpoint_resource_group_name = azurerm_resource_group.eventhub_observability_rg.name + + private_dns_zones = { + id = [data.azurerm_private_dns_zone.eventhub.id] + name = [data.azurerm_private_dns_zone.eventhub.name] + resource_group_name = data.azurerm_resource_group.rg_event_private_dns_zone.name + } + + private_dns_zone_record_A_name = "${var.domain}.${var.location_short_itn}" + + action = [ + { + action_group_id = data.azurerm_monitor_action_group.slack.id + webhook_properties = null + }, + { + action_group_id = data.azurerm_monitor_action_group.email.id + webhook_properties = null + } + ] + + metric_alerts_create = var.ehns_alerts_enabled + # metric_alerts = var.ehns_metric_alerts_gpd + + tags = var.tags +} + +# +# CONFIGURATION +# +module "eventhub_observability_gpd_configuration" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration?ref=v8.22.0" + + event_hub_namespace_name = module.eventhub_namespace_observability_gpd.name + event_hub_namespace_resource_group_name = azurerm_resource_group.eventhub_observability_rg.name + + eventhubs = var.eventhubs_gpd +} + +resource "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" { + name = "cdc-gpd-connection-string" + namespace_name = module.eventhub_namespace_observability_gpd.name + resource_group_name = azurerm_resource_group.eventhub_observability_rg.name + listen = true + send = true + manage = false +} + diff --git a/src/domains/observability/99_variables.tf b/src/domains/observability/99_variables.tf index 3f13914893..f6253b1bd0 100644 --- a/src/domains/observability/99_variables.tf +++ b/src/domains/observability/99_variables.tf @@ -268,6 +268,41 @@ EOD })) } +variable "ehns_metric_alerts_gpd" { + default = {} + + description = <= ["10.3.14.64/27"] +cidr_subnet_observability_gpd_evh = ["10.3.14.64/27"] # = ["10.3.14.96/27"] # = ["10.3.14.128/27"] # = ["10.3.14.160/27"] @@ -260,3 +260,170 @@ ehns_metric_alerts = { ], }, } + +eventhubs_gpd = [ + { + name = "connect-cluster-offsets" # debezium internal use + partitions = 1 + message_retention = 1 + consumers = ["connect-cluster-offsets"] + keys = [ + { + name = "connect-cluster-offsets" + listen = true + send = true + manage = false + } + ] + }, + { + name = "connect-cluster-status" # debezium internal use + partitions = 1 + message_retention = 1 + consumers = ["connect-cluster-offsets"] + keys = [ + { + name = "connect-cluster-status" + listen = true + send = true + manage = false + } + ] + }, + { + name = "connect-cluster-configs" # debezium internal use + partitions = 1 + message_retention = 1 + consumers = ["connect-cluster-configs"] + keys = [ + { + name = "connect-cluster-configs" + listen = true + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_option" + partitions = 1 + message_retention = 1 + consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] + keys = [ + { + name = "gpd-ingestion.apd.payment_option-rx-dl" + listen = true + send = false + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_option_metadata" + partitions = 1 + message_retention = 1 + consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" + listen = true + send = false + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_position" + partitions = 1 + message_retention = 1 + consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.payment_position-rx-dl" + listen = true + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.transfer" + partitions = 1 + message_retention = 1 + consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.transfer-rx-dl" + listen = true + send = false + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.transfer_metadata" + partitions = 1 + message_retention = 1 + consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.transfer_metadata-rx-dl" + listen = true + send = false + manage = false + } + ] + }, +] + + +# alert evh +# ehns_metric_alerts_gpd = { +# no_trx = { +# aggregation = "Total" +# metric_name = "IncomingMessages" +# description = "No transactions received from acquirer in the last 24h" +# operator = "LessThanOrEqual" +# threshold = 1000 +# frequency = "PT1H" +# window_size = "P1D" +# dimension = [ +# { +# name = "EntityName" +# operator = "Include" +# values = ["gec-ingestion-bundles-evt-tx", "gec-ingestion-cibundles-evt-tx", "gec-ingestion-paymenttypes-evt-tx", "gec-ingestion-touchpoints-evt-tx"] +# } +# ], +# }, +# active_connections = { +# aggregation = "Average" +# metric_name = "ActiveConnections" +# description = null +# operator = "LessThanOrEqual" +# threshold = 0 +# frequency = "PT5M" +# window_size = "PT15M" +# dimension = [], +# }, +# error_trx = { +# aggregation = "Total" +# metric_name = "IncomingMessages" +# description = "Transactions rejected from one acquirer file received. trx write on eventhub. check immediately" +# operator = "GreaterThan" +# threshold = 0 +# frequency = "PT5M" +# window_size = "PT30M" +# dimension = [ +# { +# name = "EntityName" +# operator = "Include" +# values = [ +# "gec-ingestion-bundles-evt-rx-pdnd", +# "gec-ingestion-cibundles-evt-rx-pdnd", +# "gec-ingestion-paymenttypes-evt-rx-pdnd", +# "gec-ingestion-touchpoints-evt-rx-pdnd" +# ] +# } +# ], +# }, +# } diff --git a/src/domains/observability/env/prod/terraform.tfvars b/src/domains/observability/env/prod/terraform.tfvars index 89d6dd4d04..04c5611268 100644 --- a/src/domains/observability/env/prod/terraform.tfvars +++ b/src/domains/observability/env/prod/terraform.tfvars @@ -56,11 +56,12 @@ apim_dns_zone_prefix = "NOT_USED" enable_sa_backup = true cidr_subnet_observability_storage = ["10.3.14.0/27"] cidr_subnet_observability_evh = ["10.3.14.32/27"] -# = ["10.3.14.64/27"] +cidr_subnet_observability_gpd_evh = ["10.3.14.64/27"] # = ["10.3.14.96/27"] # = ["10.3.14.128/27"] # = ["10.3.14.160/27"] + # # EventHub # @@ -259,3 +260,170 @@ ehns_metric_alerts = { ], }, } + +eventhubs_gpd = [ + { + name = "connect-cluster-offsets" # debezium internal use + partitions = 32 + message_retention = 7 + consumers = ["connect-cluster-offsets"] + keys = [ + { + name = "connect-cluster-offsets" + listen = true + send = true + manage = false + } + ] + }, + { + name = "connect-cluster-status" # debezium internal use + partitions = 32 + message_retention = 7 + consumers = ["connect-cluster-offsets"] + keys = [ + { + name = "connect-cluster-status" + listen = true + send = true + manage = false + } + ] + }, + { + name = "connect-cluster-configs" # debezium internal use + partitions = 32 + message_retention = 7 + consumers = ["connect-cluster-configs"] + keys = [ + { + name = "connect-cluster-configs" + listen = true + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_option" + partitions = 32 + message_retention = 7 + consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] + keys = [ + { + name = "gpd-ingestion.apd.payment_option-rx-dl" + listen = true + send = false + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_option_metadata" + partitions = 32 + message_retention = 7 + consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" + listen = true + send = false + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_position" + partitions = 32 + message_retention = 7 + consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.payment_position-rx-dl" + listen = true + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.transfer" + partitions = 32 + message_retention = 7 + consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.transfer-rx-dl" + listen = true + send = false + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.transfer_metadata" + partitions = 32 + message_retention = 7 + consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.transfer_metadata-rx-dl" + listen = true + send = false + manage = false + } + ] + }, +] + + +# alert evh +# ehns_metric_alerts_gpd = { +# no_trx = { +# aggregation = "Total" +# metric_name = "IncomingMessages" +# description = "No transactions received from acquirer in the last 24h" +# operator = "LessThanOrEqual" +# threshold = 1000 +# frequency = "PT1H" +# window_size = "P1D" +# dimension = [ +# { +# name = "EntityName" +# operator = "Include" +# values = ["gec-ingestion-bundles-evt-tx", "gec-ingestion-cibundles-evt-tx", "gec-ingestion-paymenttypes-evt-tx", "gec-ingestion-touchpoints-evt-tx"] +# } +# ], +# }, +# active_connections = { +# aggregation = "Average" +# metric_name = "ActiveConnections" +# description = null +# operator = "LessThanOrEqual" +# threshold = 0 +# frequency = "PT5M" +# window_size = "PT15M" +# dimension = [], +# }, +# error_trx = { +# aggregation = "Total" +# metric_name = "IncomingMessages" +# description = "Transactions rejected from one acquirer file received. trx write on eventhub. check immediately" +# operator = "GreaterThan" +# threshold = 0 +# frequency = "PT5M" +# window_size = "PT30M" +# dimension = [ +# { +# name = "EntityName" +# operator = "Include" +# values = [ +# "gec-ingestion-bundles-evt-rx-pdnd", +# "gec-ingestion-cibundles-evt-rx-pdnd", +# "gec-ingestion-paymenttypes-evt-rx-pdnd", +# "gec-ingestion-touchpoints-evt-rx-pdnd" +# ] +# } +# ], +# }, +# } diff --git a/src/domains/observability/env/uat/terraform.tfvars b/src/domains/observability/env/uat/terraform.tfvars index cc0635d77a..3771b06cf1 100644 --- a/src/domains/observability/env/uat/terraform.tfvars +++ b/src/domains/observability/env/uat/terraform.tfvars @@ -56,11 +56,12 @@ apim_dns_zone_prefix = "uat.platform" # observability Ingestion cfg cidr_subnet_observability_storage = ["10.3.14.0/27"] cidr_subnet_observability_evh = ["10.3.14.32/27"] -# = ["10.3.14.64/27"] +cidr_subnet_observability_gpd_evh = ["10.3.14.64/27"] # = ["10.3.14.96/27"] # = ["10.3.14.128/27"] # = ["10.3.14.160/27"] + # # EventHub # @@ -259,3 +260,170 @@ ehns_metric_alerts = { ], }, } + +eventhubs_gpd = [ + { + name = "connect-cluster-offsets" # debezium internal use + partitions = 1 + message_retention = 1 + consumers = ["connect-cluster-offsets"] + keys = [ + { + name = "connect-cluster-offsets" + listen = true + send = true + manage = false + } + ] + }, + { + name = "connect-cluster-status" # debezium internal use + partitions = 1 + message_retention = 1 + consumers = ["connect-cluster-offsets"] + keys = [ + { + name = "connect-cluster-status" + listen = true + send = true + manage = false + } + ] + }, + { + name = "connect-cluster-configs" # debezium internal use + partitions = 1 + message_retention = 1 + consumers = ["connect-cluster-configs"] + keys = [ + { + name = "connect-cluster-configs" + listen = true + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_option" + partitions = 1 + message_retention = 1 + consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] + keys = [ + { + name = "gpd-ingestion.apd.payment_option-rx-dl" + listen = true + send = false + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_option_metadata" + partitions = 1 + message_retention = 1 + consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" + listen = true + send = false + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_position" + partitions = 1 + message_retention = 1 + consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.payment_position-rx-dl" + listen = true + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.transfer" + partitions = 1 + message_retention = 1 + consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.transfer-rx-dl" + listen = true + send = false + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.transfer_metadata" + partitions = 1 + message_retention = 1 + consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] + keys = [ + { + name = "gpd-ingestion.apd.transfer_metadata-rx-dl" + listen = true + send = false + manage = false + } + ] + }, +] + + +# alert evh +# ehns_metric_alerts_gpd = { +# no_trx = { +# aggregation = "Total" +# metric_name = "IncomingMessages" +# description = "No transactions received from acquirer in the last 24h" +# operator = "LessThanOrEqual" +# threshold = 1000 +# frequency = "PT1H" +# window_size = "P1D" +# dimension = [ +# { +# name = "EntityName" +# operator = "Include" +# values = ["gec-ingestion-bundles-evt-tx", "gec-ingestion-cibundles-evt-tx", "gec-ingestion-paymenttypes-evt-tx", "gec-ingestion-touchpoints-evt-tx"] +# } +# ], +# }, +# active_connections = { +# aggregation = "Average" +# metric_name = "ActiveConnections" +# description = null +# operator = "LessThanOrEqual" +# threshold = 0 +# frequency = "PT5M" +# window_size = "PT15M" +# dimension = [], +# }, +# error_trx = { +# aggregation = "Total" +# metric_name = "IncomingMessages" +# description = "Transactions rejected from one acquirer file received. trx write on eventhub. check immediately" +# operator = "GreaterThan" +# threshold = 0 +# frequency = "PT5M" +# window_size = "PT30M" +# dimension = [ +# { +# name = "EntityName" +# operator = "Include" +# values = [ +# "gec-ingestion-bundles-evt-rx-pdnd", +# "gec-ingestion-cibundles-evt-rx-pdnd", +# "gec-ingestion-paymenttypes-evt-rx-pdnd", +# "gec-ingestion-touchpoints-evt-rx-pdnd" +# ] +# } +# ], +# }, +# } From c50f0dc463c0cd5276c17eb144646e4a55bd3fdf Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 28 Oct 2024 17:41:48 +0100 Subject: [PATCH 07/55] fix --- .../gpdingestion-secrets/.terraform.lock.hcl | 107 ------ .../gpdingestion-secrets/00_azuread.tf | 16 - .../gpdingestion-secrets/01_keyvault.tf | 101 ------ src/domains/gpdingestion-secrets/02_azdo.tf | 23 -- .../gpdingestion-secrets/02_init_sops.tf | 21 -- .../gpdingestion-secrets/03_sops_secrets.tf | 54 --- src/domains/gpdingestion-secrets/99_locals.tf | 11 - src/domains/gpdingestion-secrets/99_main.tf | 47 --- .../gpdingestion-secrets/99_variables.tf | 101 ------ src/domains/gpdingestion-secrets/README.md | 65 ---- .../env/itn-dev/backend.ini | 1 - .../env/itn-dev/backend.tfvars | 4 - .../env/itn-dev/terraform.tfvars | 30 -- .../env/itn-prod/backend.ini | 1 - .../env/itn-prod/backend.tfvars | 4 - .../env/itn-prod/terraform.tfvars | 30 -- .../env/itn-uat/backend.ini | 1 - .../env/itn-uat/backend.tfvars | 4 - .../env/itn-uat/terraform.tfvars | 27 -- .../secret/itn-dev/configs.json | 1 - .../secret/itn-dev/secret.ini | 3 - .../secret/itn-prod/configs.json | 2 - .../secret/itn-prod/secret.ini | 3 - .../secret/itn-uat/configs.json | 2 - .../secret/itn-uat/secret.ini | 3 - src/domains/gpdingestion-secrets/sops.sh | 137 -------- src/domains/gpdingestion-secrets/terraform.sh | 324 ------------------ src/domains/gpdingestion-secrets/terrasops.sh | 29 -- 28 files changed, 1152 deletions(-) delete mode 100644 src/domains/gpdingestion-secrets/.terraform.lock.hcl delete mode 100644 src/domains/gpdingestion-secrets/00_azuread.tf delete mode 100644 src/domains/gpdingestion-secrets/01_keyvault.tf delete mode 100644 src/domains/gpdingestion-secrets/02_azdo.tf delete mode 100644 src/domains/gpdingestion-secrets/02_init_sops.tf delete mode 100644 src/domains/gpdingestion-secrets/03_sops_secrets.tf delete mode 100644 src/domains/gpdingestion-secrets/99_locals.tf delete mode 100644 src/domains/gpdingestion-secrets/99_main.tf delete mode 100644 src/domains/gpdingestion-secrets/99_variables.tf delete mode 100644 src/domains/gpdingestion-secrets/README.md delete mode 100644 src/domains/gpdingestion-secrets/env/itn-dev/backend.ini delete mode 100644 src/domains/gpdingestion-secrets/env/itn-dev/backend.tfvars delete mode 100644 src/domains/gpdingestion-secrets/env/itn-dev/terraform.tfvars delete mode 100644 src/domains/gpdingestion-secrets/env/itn-prod/backend.ini delete mode 100644 src/domains/gpdingestion-secrets/env/itn-prod/backend.tfvars delete mode 100644 src/domains/gpdingestion-secrets/env/itn-prod/terraform.tfvars delete mode 100644 src/domains/gpdingestion-secrets/env/itn-uat/backend.ini delete mode 100644 src/domains/gpdingestion-secrets/env/itn-uat/backend.tfvars delete mode 100644 src/domains/gpdingestion-secrets/env/itn-uat/terraform.tfvars delete mode 100644 src/domains/gpdingestion-secrets/secret/itn-dev/configs.json delete mode 100644 src/domains/gpdingestion-secrets/secret/itn-dev/secret.ini delete mode 100644 src/domains/gpdingestion-secrets/secret/itn-prod/configs.json delete mode 100644 src/domains/gpdingestion-secrets/secret/itn-prod/secret.ini delete mode 100644 src/domains/gpdingestion-secrets/secret/itn-uat/configs.json delete mode 100644 src/domains/gpdingestion-secrets/secret/itn-uat/secret.ini delete mode 100755 src/domains/gpdingestion-secrets/sops.sh delete mode 100755 src/domains/gpdingestion-secrets/terraform.sh delete mode 100644 src/domains/gpdingestion-secrets/terrasops.sh diff --git a/src/domains/gpdingestion-secrets/.terraform.lock.hcl b/src/domains/gpdingestion-secrets/.terraform.lock.hcl deleted file mode 100644 index a389468af5..0000000000 --- a/src/domains/gpdingestion-secrets/.terraform.lock.hcl +++ /dev/null @@ -1,107 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/azuread" { - version = "2.47.0" - constraints = "<= 2.47.0" - hashes = [ - "h1:g8+gBFM4QVOEQFqAEs5pR6iXpbGvgPvcEi1evHwziyw=", - "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", - "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", - "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", - "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", - "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", - "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", - "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", - "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", - "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", - "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", - "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", - "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", - "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", - ] -} - -provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.106.0" - constraints = "~> 3.30, <= 3.106.0" - hashes = [ - "h1:6t9Nz9tYAR9BfHZ8yc56m+GKRl0nriwjQ5DyA0/TnCs=", - "h1:Mxe1/I27IZK3BP6cm84Gt0+7PXd2EDaDUMxuljm/rUA=", - "zh:07980d6fdc40c0adb670c8413a5c667917d6dbb51fcedc467c35d64c2f3a1f47", - "zh:2e6e8491b1f089644b0d23f8da83398f1e10cf5a62b16efcef2b5454fe923038", - "zh:450dbd72821c5619cc3bcdc20fdd0e29515147e44b733f9c79d3a75851810055", - "zh:5e234c0a2f3c9677ea72b2a6e6ca90defb99fab29ae565f5d1f70728ba4ba78f", - "zh:83fd042ece6977429d79affd03d6ce963d2f122604dbf15a1abf203d7a7bbc8a", - "zh:93027e1f66b3bf83398d572d4e6f6e7777330c78c54da3226dadd50fd868ada9", - "zh:ae3d1dd66140c303df97d93c47a60f16735ce17cf156f45475dcee4a7360af5b", - "zh:daf9d2eb89e785458a76b88bf2ef0696c472094c77cc9cff3b3ea4b885c5a482", - "zh:dd46370141651e6549da6d85e25c7a6770c47581bbaaa27eda2886d41d849747", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f77405c0d8f6e0d93d9da83256b3b02c164bad4c791ed9604310ff02ae086ad1", - "zh:ffa769147bda833aef8802e3a391bd175ec749862764d61cbdaa8200d5b8f893", - ] -} - -provider "registry.terraform.io/hashicorp/external" { - version = "2.2.3" - constraints = "<= 2.2.3" - hashes = [ - "h1:648ZjJR81c2W1OLtYmUQa9/1rGr3vvZSuX9dR1ucGWY=", - "h1:D2RKjqoU26isFINpmeKG9NS0LvkPmrQkNXeYO2TdgyA=", - "zh:184ecd339d764de845db0e5b8a9c87893dcd0c9d822167f73658f89d80ec31c9", - "zh:2661eaca31d17d6bbb18a8f673bbfe3fe1b9b7326e60d0ceb302017003274e3c", - "zh:2c0a180f6d1fc2ba6e03f7dfc5f73b617e45408681f75bca75aa82f3796df0e4", - "zh:4b92ae44c6baef4c4952c47be00541055cb5280dd3bc8031dba5a1b2ee982387", - "zh:5641694d5daf3893d7ea90be03b6fa575211a08814ffe70998d5adb8b59cdc0a", - "zh:5bd55a2be8a1c20d732ac9c604b839e1cadc8c49006315dffa4d709b6874df32", - "zh:6e0ef5d11e1597202424b7d69b9da7b881494c9b13a3d4026fc47012dc651c79", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9e19f89fa25004d3b926a8d15ea630b4bde62f1fa4ed5e11a3d27aabddb77353", - "zh:b763efdd69fd097616b4a4c89cf333b4cee9699ac6432d73d2756f8335d1213f", - "zh:e3b561efdee510b2b445f76a52a902c52bee8e13095e7f4bed7c80f10f8d294a", - "zh:fe660bb8781ee043a093b9a20e53069974475dcaa5791a1f45fd03c61a26478a", - ] -} - -provider "registry.terraform.io/hashicorp/kubernetes" { - version = "2.16.1" - constraints = "<= 2.16.1" - hashes = [ - "h1:PO4Ye/+lu5hCaUEOtwNOldQYoA0dqL1bcBICIpdlcd8=", - "h1:kO/d+ZMZYM2tNMMFHZqBmVR0MeemoGnI2G2NSN92CrU=", - "zh:06224975f5910d41e73b35a4d5079861da2c24f9353e3ebb015fbb3b3b996b1c", - "zh:2bc400a8d9fe7755cca27c2551564a9e2609cfadc77f526ef855114ee02d446f", - "zh:3a479014187af1d0aec3a1d3d9c09551b801956fe6dd29af1186dec86712731b", - "zh:73fb0a69f1abdb02858b6589f7fab6d989a0f422f7ad95ed662aaa84872d3473", - "zh:a33852cd382cbc8e06d3f6c018b468ad809d24d912d64722e037aed1f9bf39db", - "zh:b533ff2214dca90296b1d22eace7eaa7e3efe5a7ae9da66a112094abc932db4f", - "zh:ddf74d8bb1aeb01dc2c36ef40e2b283d32b2a96db73f6daaf179fa2f10949c80", - "zh:e720f3a15d34e795fa9ff90bc755e838ebb4aef894aa2a423fb16dfa6d6b0667", - "zh:e789ae70a658800cb0a19ef7e4e9b26b5a38a92b43d1f41d64fc8bb46539cefb", - "zh:e8aed7dc0bd8f843d607dee5f72640dbef6835a8b1c6ea12cea5b4ec53e463f7", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:fb3ac4f43c8b0dfc0b0103dd0f062ea72b3a34518d4c8808e3a44c9a3dd5f024", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.1" - constraints = "~> 3.2, <= 3.2.1" - hashes = [ - "h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=", - "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", - "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", - "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", - "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", - "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", - "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", - "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", - "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", - "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", - "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", - "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", - ] -} diff --git a/src/domains/gpdingestion-secrets/00_azuread.tf b/src/domains/gpdingestion-secrets/00_azuread.tf deleted file mode 100644 index 14a0893a9f..0000000000 --- a/src/domains/gpdingestion-secrets/00_azuread.tf +++ /dev/null @@ -1,16 +0,0 @@ -# Azure AD -data "azuread_group" "adgroup_admin" { - display_name = "${local.product}-adgroup-admin" -} - -data "azuread_group" "adgroup_developers" { - display_name = "${local.product}-adgroup-developers" -} - -data "azuread_group" "adgroup_externals" { - display_name = "${local.product}-adgroup-externals" -} - -data "azuread_group" "adgroup_security" { - display_name = "${local.product}-adgroup-security" -} \ No newline at end of file diff --git a/src/domains/gpdingestion-secrets/01_keyvault.tf b/src/domains/gpdingestion-secrets/01_keyvault.tf deleted file mode 100644 index be275a9653..0000000000 --- a/src/domains/gpdingestion-secrets/01_keyvault.tf +++ /dev/null @@ -1,101 +0,0 @@ -resource "azurerm_resource_group" "sec_rg" { - name = "${local.product}-${var.location_short}-${var.domain}-sec-rg" - location = var.location - - tags = var.tags -} - -module "key_vault" { - source = "./.terraform/modules/__v3__/key_vault" - - name = "${local.product}-${var.location_short}-${var.domain}-kv" - location = azurerm_resource_group.sec_rg.location - resource_group_name = azurerm_resource_group.sec_rg.name - tenant_id = data.azurerm_client_config.current.tenant_id - soft_delete_retention_days = 90 - - tags = var.tags -} - -## ad group policy ## -resource "azurerm_key_vault_access_policy" "ad_group_policy" { - key_vault_id = module.key_vault.id - - tenant_id = data.azurerm_client_config.current.tenant_id - object_id = data.azuread_group.adgroup_admin.object_id - - key_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Encrypt", "Decrypt", "Backup", "Purge", "Recover", "Restore", "Sign", "UnwrapKey", "Update", "Verify", "WrapKey", "Release", "Rotate", "GetRotationPolicy", "SetRotationPolicy"] - secret_permissions = ["Get", "List", "Set", "Delete", "Backup", "Purge", "Recover", "Restore"] - storage_permissions = [] - certificate_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Restore", "Purge", "Recover", ] -} - -## ad group policy ## -resource "azurerm_key_vault_access_policy" "adgroup_developers_policy" { - count = var.env_short != "p" ? 1 : 0 - - key_vault_id = module.key_vault.id - - tenant_id = data.azurerm_client_config.current.tenant_id - object_id = data.azuread_group.adgroup_developers.object_id - - key_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Encrypt", "Decrypt", "Recover", "Rotate", "GetRotationPolicy"] - secret_permissions = ["Get", "List", "Set", "Delete", "Recover", ] - storage_permissions = [] - certificate_permissions = [ - "Get", "List", "Update", "Create", "Import", - "Delete", "Restore", "Purge", "Recover" - ] -} - -## ad group policy ## -resource "azurerm_key_vault_access_policy" "adgroup_externals_policy" { - count = var.env_short != "p" ? 1 : 0 - - key_vault_id = module.key_vault.id - - tenant_id = data.azurerm_client_config.current.tenant_id - object_id = data.azuread_group.adgroup_externals.object_id - - key_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Encrypt", "Decrypt", "Recover", "Rotate", "GetRotationPolicy"] - secret_permissions = ["Get", "List", "Set", "Delete", "Recover", ] - storage_permissions = [] - certificate_permissions = [ - "Get", "List", "Update", "Create", "Import", - "Delete", "Restore", "Purge", "Recover" - ] -} - -## ad group policy ## -data "azuread_service_principal" "iac_principal" { - count = var.enable_iac_pipeline ? 1 : 0 - display_name = "pagopaspa-pagoPA-iac-${data.azurerm_subscription.current.subscription_id}" -} - -resource "azurerm_key_vault_access_policy" "azdevops_iac_policy" { - count = var.enable_iac_pipeline ? 1 : 0 - key_vault_id = module.key_vault.id - tenant_id = data.azurerm_client_config.current.tenant_id - object_id = data.azuread_service_principal.iac_principal[0].object_id - - secret_permissions = ["Get", "List", "Set", ] - certificate_permissions = ["SetIssuers", "DeleteIssuers", "Purge", "List", "Get"] - key_permissions = ["Get", "List", "Update", "Create", "Import", "Delete", "Encrypt", "Decrypt"] - - storage_permissions = [] -} - -################ -## Secrets ## -################ - -# create json letsencrypt inside kv -# requierd: Docker -module "letsencrypt_gpdingestion" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git///letsencrypt_credential?ref=v8.44.0" - - prefix = var.prefix - env = var.env_short - key_vault_name = module.key_vault.name - subscription_name = local.subscription_name -} diff --git a/src/domains/gpdingestion-secrets/02_azdo.tf b/src/domains/gpdingestion-secrets/02_azdo.tf deleted file mode 100644 index 5683ffec89..0000000000 --- a/src/domains/gpdingestion-secrets/02_azdo.tf +++ /dev/null @@ -1,23 +0,0 @@ -# -# Policy -# - -data "azurerm_user_assigned_identity" "iac_federated_azdo" { - for_each = local.azdo_iac_managed_identities - name = each.key - resource_group_name = local.azdo_managed_identity_rg_name -} - -resource "azurerm_key_vault_access_policy" "azdevops_iac_managed_identities" { - for_each = local.azdo_iac_managed_identities - - key_vault_id = module.key_vault.id - tenant_id = data.azurerm_client_config.current.tenant_id - object_id = data.azurerm_user_assigned_identity.iac_federated_azdo[each.key].principal_id - - secret_permissions = ["Get", "List", "Set", ] - - certificate_permissions = ["SetIssuers", "DeleteIssuers", "Purge", "List", "Get"] - - storage_permissions = [] -} diff --git a/src/domains/gpdingestion-secrets/02_init_sops.tf b/src/domains/gpdingestion-secrets/02_init_sops.tf deleted file mode 100644 index e93d0651a0..0000000000 --- a/src/domains/gpdingestion-secrets/02_init_sops.tf +++ /dev/null @@ -1,21 +0,0 @@ -moved { - from = azurerm_key_vault_key.generated - to = azurerm_key_vault_key.sops_key -} - -resource "azurerm_key_vault_key" "sops_key" { - name = "${local.product}-${var.domain}-sops-key" - key_vault_id = module.key_vault.id - key_type = "RSA" - key_size = 2048 - - key_opts = [ - "decrypt", - "encrypt", - ] - - depends_on = [ - azurerm_key_vault_access_policy.adgroup_developers_policy, - azurerm_key_vault_access_policy.ad_group_policy, - ] -} diff --git a/src/domains/gpdingestion-secrets/03_sops_secrets.tf b/src/domains/gpdingestion-secrets/03_sops_secrets.tf deleted file mode 100644 index 68c06265a2..0000000000 --- a/src/domains/gpdingestion-secrets/03_sops_secrets.tf +++ /dev/null @@ -1,54 +0,0 @@ -moved { - from = data.external.external2 - to = data.external.terrasops -} - -data "external" "terrasops" { - program = [ - "bash", "terrasops.sh" - ] - query = { - env = "${var.location_short}-${var.env}" - } - -} - -locals { - all_enc_secrets_value = can(data.external.terrasops.result) ? flatten([ - for k, v in data.external.terrasops.result : { - valore = v - chiave = k - } - ]) : [] - - config_secret_data = jsondecode(file(var.input_file)) - all_config_secrets_value = flatten([ - for kc, vc in local.config_secret_data : { - valore = vc - chiave = kc - } - ]) - - all_secrets_value = concat(local.all_config_secrets_value, local.all_enc_secrets_value) -} - -## SOPS secrets - -## Upload all encrypted secrets -resource "azurerm_key_vault_secret" "secret" { - for_each = { for i, v in local.all_secrets_value : local.all_secrets_value[i].chiave => i } - - key_vault_id = module.key_vault.id - name = local.all_secrets_value[each.value].chiave - value = local.all_secrets_value[each.value].valore - - depends_on = [ - module.key_vault, - azurerm_key_vault_key.sops_key, - data.external.terrasops, - azurerm_key_vault_access_policy.adgroup_developers_policy, - azurerm_key_vault_access_policy.ad_group_policy, - ] -} - -# ⚠️ The secrets from resources are set in gpdingestion-app to avoid circular dependency diff --git a/src/domains/gpdingestion-secrets/99_locals.tf b/src/domains/gpdingestion-secrets/99_locals.tf deleted file mode 100644 index 084cb86f0c..0000000000 --- a/src/domains/gpdingestion-secrets/99_locals.tf +++ /dev/null @@ -1,11 +0,0 @@ -locals { - project = "${var.prefix}-${var.env_short}-${var.location_short}-${var.domain}" - product = "${var.prefix}-${var.env_short}" - - - subscription_name = "${var.env}-${var.prefix}" - - azdo_managed_identity_rg_name = "pagopa-${var.env_short}-identity-rg" - azdo_iac_managed_identities = toset(["azdo-${var.env}-pagopa-iac-deploy", "azdo-${var.env}-pagopa-iac-plan"]) - -} diff --git a/src/domains/gpdingestion-secrets/99_main.tf b/src/domains/gpdingestion-secrets/99_main.tf deleted file mode 100644 index ba2156d6dd..0000000000 --- a/src/domains/gpdingestion-secrets/99_main.tf +++ /dev/null @@ -1,47 +0,0 @@ -terraform { - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "<= 3.116.0" - } - azuread = { - source = "hashicorp/azuread" - version = "<= 3.0.2" - } - null = { - source = "hashicorp/null" - version = "<= 3.2.3" - } - external = { - source = "hashicorp/external" - version = "<= 2.3.4" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = "<= 2.33.0" - } - } - - backend "azurerm" {} -} - -provider "azurerm" { - features { - key_vault { - purge_soft_delete_on_destroy = false - } - } -} - -provider "kubernetes" { - config_path = "~/.kube/config-${var.prefix}-${var.env_short}-${var.location_short}-${var.env}-aks" - config_context = "${var.prefix}-${var.env_short}-${var.location_short}-${var.env}-aks" -} - -data "azurerm_subscription" "current" {} - -data "azurerm_client_config" "current" {} - -module "__v3__" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" -} diff --git a/src/domains/gpdingestion-secrets/99_variables.tf b/src/domains/gpdingestion-secrets/99_variables.tf deleted file mode 100644 index 3a7cff7fcf..0000000000 --- a/src/domains/gpdingestion-secrets/99_variables.tf +++ /dev/null @@ -1,101 +0,0 @@ -# general - -variable "prefix" { - type = string - validation { - condition = ( - length(var.prefix) <= 6 - ) - error_message = "Max length is 6 chars." - } -} - -variable "env" { - type = string -} - -variable "env_short" { - type = string - validation { - condition = ( - length(var.env_short) == 1 - ) - error_message = "Length must be 1 chars." - } -} - -variable "domain" { - type = string - validation { - condition = ( - length(var.domain) <= 12 - ) - error_message = "Max length is 12 chars." - } -} - -variable "location" { - type = string - description = "One of westeurope, northeurope" -} - -variable "location_short" { - type = string - validation { - condition = ( - length(var.location_short) == 3 - ) - error_message = "Length must be 3 chars." - } - description = "One of weu, itn" -} - -variable "instance" { - type = string - description = "One of beta, prod01, prod02" -} - -variable "tags" { - type = map(any) - default = { - CreatedBy = "Terraform" - } -} - -### - -variable "input_file" { - type = string - description = "secret json file" -} - -variable "enable_iac_pipeline" { - type = bool - description = "If true create the key vault policy to allow used by azure devops iac pipelines." - default = false -} - - -variable "kv-key-permissions-read" { - type = list(string) - description = "List of read key permissions" - default = ["Get", "List"] -} - -variable "kv-secret-permissions-read" { - type = list(string) - description = "List of read secret permissions" - default = ["Get", "List"] -} - -variable "kv-certificate-permissions-read" { - type = list(string) - description = "List of read certificate permissions" - default = ["Get", "GetIssuers", "List", "ListIssuers"] -} - -variable "kv-storage-permissions-read" { - type = list(string) - description = "List of read storage permissions" - default = ["Get", "GetSAS", "List", "ListSAS"] -} diff --git a/src/domains/gpdingestion-secrets/README.md b/src/domains/gpdingestion-secrets/README.md deleted file mode 100644 index ccd6f2a60c..0000000000 --- a/src/domains/gpdingestion-secrets/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# paymentoptions-secrets - - -## Requirements - -| Name | Version | -|------|---------| -| [azuread](#requirement\_azuread) | <= 2.47.0 | -| [azurerm](#requirement\_azurerm) | <= 3.106.0 | -| [external](#requirement\_external) | <= 2.2.3 | -| [kubernetes](#requirement\_kubernetes) | <= 2.16.1 | -| [null](#requirement\_null) | <= 3.2.1 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [key\_vault](#module\_key\_vault) | git::https://github.com/pagopa/terraform-azurerm-v3.git//key_vault | v8.22.0 | -| [letsencrypt\_paymentoptions](#module\_letsencrypt\_paymentoptions) | git::https://github.com/pagopa/terraform-azurerm-v3.git///letsencrypt_credential | v8.44.0 | - -## Resources - -| Name | Type | -|------|------| -| [azurerm_key_vault_access_policy.ad_group_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | -| [azurerm_key_vault_access_policy.adgroup_developers_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | -| [azurerm_key_vault_access_policy.adgroup_externals_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | -| [azurerm_key_vault_access_policy.azdevops_iac_managed_identities](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | -| [azurerm_key_vault_access_policy.azdevops_iac_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | -| [azurerm_key_vault_key.sops_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_key) | resource | -| [azurerm_key_vault_secret.secret](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_resource_group.sec_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | -| [azuread_group.adgroup_admin](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | -| [azuread_group.adgroup_developers](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | -| [azuread_group.adgroup_externals](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | -| [azuread_group.adgroup_security](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | -| [azuread_service_principal.iac_principal](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/service_principal) | data source | -| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | -| [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | -| [azurerm_user_assigned_identity.iac_federated_azdo](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/user_assigned_identity) | data source | -| [external_external.terrasops](https://registry.terraform.io/providers/hashicorp/external/latest/docs/data-sources/external) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [domain](#input\_domain) | n/a | `string` | n/a | yes | -| [enable\_iac\_pipeline](#input\_enable\_iac\_pipeline) | If true create the key vault policy to allow used by azure devops iac pipelines. | `bool` | `false` | no | -| [env](#input\_env) | n/a | `string` | n/a | yes | -| [env\_short](#input\_env\_short) | n/a | `string` | n/a | yes | -| [input\_file](#input\_input\_file) | secret json file | `string` | n/a | yes | -| [instance](#input\_instance) | One of beta, prod01, prod02 | `string` | n/a | yes | -| [kv-certificate-permissions-read](#input\_kv-certificate-permissions-read) | List of read certificate permissions | `list(string)` | 
[
  "Get",
  "GetIssuers",
  "List",
  "ListIssuers"
]
| no | -| [kv-key-permissions-read](#input\_kv-key-permissions-read) | List of read key permissions | `list(string)` | 
[
  "Get",
  "List"
]
| no | -| [kv-secret-permissions-read](#input\_kv-secret-permissions-read) | List of read secret permissions | `list(string)` | 
[
  "Get",
  "List"
]
| no | -| [kv-storage-permissions-read](#input\_kv-storage-permissions-read) | List of read storage permissions | `list(string)` | 
[
  "Get",
  "GetSAS",
  "List",
  "ListSAS"
]
| no | -| [location](#input\_location) | One of westeurope, northeurope | `string` | n/a | yes | -| [location\_short](#input\_location\_short) | One of weu, itn | `string` | n/a | yes | -| [prefix](#input\_prefix) | n/a | `string` | n/a | yes | -| [tags](#input\_tags) | n/a | `map(any)` | 
{
  "CreatedBy": "Terraform"
}
| no | - -## Outputs - -No outputs. - diff --git a/src/domains/gpdingestion-secrets/env/itn-dev/backend.ini b/src/domains/gpdingestion-secrets/env/itn-dev/backend.ini deleted file mode 100644 index f3ea2d530c..0000000000 --- a/src/domains/gpdingestion-secrets/env/itn-dev/backend.ini +++ /dev/null @@ -1 +0,0 @@ -subscription=DEV-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-secrets/env/itn-dev/backend.tfvars b/src/domains/gpdingestion-secrets/env/itn-dev/backend.tfvars deleted file mode 100644 index dfd890cc01..0000000000 --- a/src/domains/gpdingestion-secrets/env/itn-dev/backend.tfvars +++ /dev/null @@ -1,4 +0,0 @@ -resource_group_name = "terraform-state-rg" -storage_account_name = "tfinfdevpagopa" -container_name = "terraform-state" -key = "gpdingestion-secret-dev.terraform.tfstate" diff --git a/src/domains/gpdingestion-secrets/env/itn-dev/terraform.tfvars b/src/domains/gpdingestion-secrets/env/itn-dev/terraform.tfvars deleted file mode 100644 index 3fc2637c4c..0000000000 --- a/src/domains/gpdingestion-secrets/env/itn-dev/terraform.tfvars +++ /dev/null @@ -1,30 +0,0 @@ -prefix = "pagopa" -env_short = "d" -env = "dev" -domain = "gpdingestion" -location = "italynorth" -location_short = "itn" -instance = "dev" - -tags = { - CreatedBy = "Terraform" - Environment = "Dev" - Owner = "pagoPA" - Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-secrets" - CostCenter = "TS310 - PAGAMENTI & SERVIZI" -} - -### External resources - -monitor_italy_resource_group_name = "pagopa-d-itn-core-monitor-rg" -log_analytics_italy_workspace_name = "pagopa-d-itn-core-law" -log_analytics_italy_workspace_resource_group_name = "pagopa-d-itn-core-monitor-rg" - -input_file = "./secret/itn-dev/configs.json" - -enable_iac_pipeline = true - - - - - diff --git a/src/domains/gpdingestion-secrets/env/itn-prod/backend.ini b/src/domains/gpdingestion-secrets/env/itn-prod/backend.ini deleted file mode 100644 index 6318425346..0000000000 --- a/src/domains/gpdingestion-secrets/env/itn-prod/backend.ini +++ /dev/null @@ -1 +0,0 @@ -subscription=PROD-pagoPA diff --git a/src/domains/gpdingestion-secrets/env/itn-prod/backend.tfvars b/src/domains/gpdingestion-secrets/env/itn-prod/backend.tfvars deleted file mode 100644 index 9277ee7b79..0000000000 --- a/src/domains/gpdingestion-secrets/env/itn-prod/backend.tfvars +++ /dev/null @@ -1,4 +0,0 @@ -resource_group_name = "terraform-state-rg" -storage_account_name = "tfinfprodpagopa" -container_name = "terraform-state" -key = "gpdingestion-secret-prod.terraform.tfstate" diff --git a/src/domains/gpdingestion-secrets/env/itn-prod/terraform.tfvars b/src/domains/gpdingestion-secrets/env/itn-prod/terraform.tfvars deleted file mode 100644 index 8f9451ab76..0000000000 --- a/src/domains/gpdingestion-secrets/env/itn-prod/terraform.tfvars +++ /dev/null @@ -1,30 +0,0 @@ -prefix = "pagopa" -env_short = "p" -env = "prod" -domain = "gpdingestion" -location = "gpdingestion" -location_short = "itn" -instance = "prod" - -tags = { - CreatedBy = "Terraform" - Environment = "Prod" - Owner = "pagoPA" - Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-secrets" - CostCenter = "TS310 - PAGAMENTI & SERVIZI" -} - -### External resources - -monitor_italy_resource_group_name = "pagopa-d-itn-core-monitor-rg" -log_analytics_italy_workspace_name = "pagopa-d-itn-core-law" -log_analytics_italy_workspace_resource_group_name = "pagopa-d-itn-core-monitor-rg" - -input_file = "./secret/itn-prod/configs.json" - -enable_iac_pipeline = true - - - - - diff --git a/src/domains/gpdingestion-secrets/env/itn-uat/backend.ini b/src/domains/gpdingestion-secrets/env/itn-uat/backend.ini deleted file mode 100644 index 1a014151dc..0000000000 --- a/src/domains/gpdingestion-secrets/env/itn-uat/backend.ini +++ /dev/null @@ -1 +0,0 @@ -subscription=UAT-pagoPA diff --git a/src/domains/gpdingestion-secrets/env/itn-uat/backend.tfvars b/src/domains/gpdingestion-secrets/env/itn-uat/backend.tfvars deleted file mode 100644 index e2a7d61cf4..0000000000 --- a/src/domains/gpdingestion-secrets/env/itn-uat/backend.tfvars +++ /dev/null @@ -1,4 +0,0 @@ -resource_group_name = "terraform-state-rg" -storage_account_name = "tfinfuatpagopa" -container_name = "terraform-state" -key = "gpdingestion-secret-uat.terraform.tfstate" diff --git a/src/domains/gpdingestion-secrets/env/itn-uat/terraform.tfvars b/src/domains/gpdingestion-secrets/env/itn-uat/terraform.tfvars deleted file mode 100644 index 7cb0ea8146..0000000000 --- a/src/domains/gpdingestion-secrets/env/itn-uat/terraform.tfvars +++ /dev/null @@ -1,27 +0,0 @@ -prefix = "pagopa" -env_short = "u" -env = "uat" -domain = "gpdingestion" -location = "italynorth" -location_short = "itn" -instance = "uat" - -tags = { - CreatedBy = "Terraform" - Environment = "Uat" - Owner = "pagoPA" - Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-secrets" - CostCenter = "TS310 - PAGAMENTI & SERVIZI" -} - -### External resources - -monitor_italy_resource_group_name = "pagopa-d-itn-core-monitor-rg" -log_analytics_italy_workspace_name = "pagopa-d-itn-core-law" -log_analytics_italy_workspace_resource_group_name = "pagopa-d-itn-core-monitor-rg" - -input_file = "./secret/itn-uat/configs.json" - -enable_iac_pipeline = true - -force = "v1" diff --git a/src/domains/gpdingestion-secrets/secret/itn-dev/configs.json b/src/domains/gpdingestion-secrets/secret/itn-dev/configs.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/src/domains/gpdingestion-secrets/secret/itn-dev/configs.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/src/domains/gpdingestion-secrets/secret/itn-dev/secret.ini b/src/domains/gpdingestion-secrets/secret/itn-dev/secret.ini deleted file mode 100644 index 5152851be5..0000000000 --- a/src/domains/gpdingestion-secrets/secret/itn-dev/secret.ini +++ /dev/null @@ -1,3 +0,0 @@ -file_crypted="noedit_secret_enc.json" -kv_name="pagopa-d-itn-gpdingestion-kv" -kv_sops_key_name="pagopa-d-gpdingestion-sops-key" diff --git a/src/domains/gpdingestion-secrets/secret/itn-prod/configs.json b/src/domains/gpdingestion-secrets/secret/itn-prod/configs.json deleted file mode 100644 index 2c63c08510..0000000000 --- a/src/domains/gpdingestion-secrets/secret/itn-prod/configs.json +++ /dev/null @@ -1,2 +0,0 @@ -{ -} diff --git a/src/domains/gpdingestion-secrets/secret/itn-prod/secret.ini b/src/domains/gpdingestion-secrets/secret/itn-prod/secret.ini deleted file mode 100644 index f4b304862a..0000000000 --- a/src/domains/gpdingestion-secrets/secret/itn-prod/secret.ini +++ /dev/null @@ -1,3 +0,0 @@ -file_crypted="noedit_secret_enc.json" -kv_name="pagopa-p-itn-gpdingestion-kv" -kv_sops_key_name="pagopa-p-gpdingestion-sops-key" diff --git a/src/domains/gpdingestion-secrets/secret/itn-uat/configs.json b/src/domains/gpdingestion-secrets/secret/itn-uat/configs.json deleted file mode 100644 index 2c63c08510..0000000000 --- a/src/domains/gpdingestion-secrets/secret/itn-uat/configs.json +++ /dev/null @@ -1,2 +0,0 @@ -{ -} diff --git a/src/domains/gpdingestion-secrets/secret/itn-uat/secret.ini b/src/domains/gpdingestion-secrets/secret/itn-uat/secret.ini deleted file mode 100644 index c31c0311f6..0000000000 --- a/src/domains/gpdingestion-secrets/secret/itn-uat/secret.ini +++ /dev/null @@ -1,3 +0,0 @@ -file_crypted="noedit_secret_enc.json" -kv_name="pagopa-u-itn-gpdingestion-kv" -kv_sops_key_name="pagopa-u-gpdingestion-sops-key" diff --git a/src/domains/gpdingestion-secrets/sops.sh b/src/domains/gpdingestion-secrets/sops.sh deleted file mode 100755 index 347b11d0ef..0000000000 --- a/src/domains/gpdingestion-secrets/sops.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash - -# set -x # Uncomment this line to enable debug mode - -# -# how to use `sh sops.sh` -# ℹ️ This script allows you to create a sops file with the relative azure key, -# it also allows you to edit the secrets and add them with the script. -# ℹ️ This script also uses an inventory file under the "./secret//secret.ini" -# directory to load environment variables. -# - -action=$1 -env=$2 -shift 2 -# shellcheck disable=SC2034 -other=( "$@" ) - -if [ -z "$action" ]; then - helpmessage=$(cat < -> decrypt json file in specified environment - example: ./sops.sh d itn-dev - example: ./sops.sh decrypt itn-dev - -./sops.sh s -> search in enc file in specified environment - example: ./sops.sh s itn-dev - example: ./sops.sh search itn-dev - -./sops.sh n -> create new file enc json template in specified environment - example: ./sops.sh n itn-dev - example: ./sops.sh new itn-dev - -./sops.sh a -> add new secret record to enc json in specified environment - example: ./sops.sh a itn-dev - example: ./sops.sh add itn-dev - -./sops.sh e -> edit enc json record in specified environment - example: ./sops.sh e itn-dev - example: ./sops.sh edit itn-dev - -./sops.sh f -> enc a json file in a specified environment - example: ./sops.sh f itn-dev - -EOF -) - echo "$helpmessage" - exit 0 -fi - -if [ -z "$env" ]; then - echo "env should be something like: itn-dev, itn-uat or itn-prod." - exit 0 -fi - -echo "🔨 Mandatory variables are correct" -file_crypted="" -kv_name="" -kv_sops_key_name="" - -# shellcheck disable=SC1090 -source "./secret/$env/secret.ini" - -echo "🔨 All variables loaded" - -# Check if kv_name and file_crypted variables are not empty -if [ -z "${kv_name}" ]; then - echo "❌ Error: kv_name variable is not defined correctly." - exit 1 -fi - -if [ -z "$file_crypted" ]; then - echo "❌ Error: file_crypted variable is not defined correctly." - exit 1 -fi - -encrypted_file_path="./secret/$env/$file_crypted" - -# Check if the key exists in the Key Vault -# shellcheck disable=SC2154 -kv_key_url=$(az keyvault key show --vault-name "$kv_name" --name "$kv_sops_key_name" --query "key.kid" -o tsv) -if [ -z "$kv_key_url" ]; then - echo "❌ The key does not exist." - exit 1 -fi -echo "[INFO] Key URL: $kv_key_url" - -echo "🔨 Key URL loaded correctly" - -if echo "d decrypt a add s search n new e edit f" | grep -w "$action" > /dev/null; then - case $action in - "d"|"decrypt") - sops --decrypt --azure-kv "$kv_key_url" "$encrypted_file_path" - if [ $? -eq 1 ]; then - echo "❌ File $encrypted_file_path NOT encrypted" - exit 0 - fi - ;; - "s"|"search") - read -r -p 'key: ' key - sops --decrypt --azure-kv "$kv_key_url" "$encrypted_file_path" | grep -i "$key" - ;; - "a"|"add") - read -r -p 'key: ' key - read -r -p 'value: ' value - sops -i --set '["'"$key"'"] "'"$value"'"' --azure-kv "$kv_key_url" "$encrypted_file_path" - echo "✅ Added key" - ;; - "n"|"new") - if [ -f "$encrypted_file_path" ]; then - echo "⚠️ file $encrypted_file_path already exists" - exit 0 - fi - echo "{}" > "$encrypted_file_path" - sops --encrypt -i --azure-kv "$kv_key_url" "$encrypted_file_path" - echo "✅ created new file for sops" - ;; - "e"|"edit") - if [ ! -f "$encrypted_file_path" ]; then - echo "⚠️ file $encrypted_file_path not found" - exit 1 - fi - - sops --azure-kv "$kv_key_url" "$encrypted_file_path" - echo "✅ edit file completed" - - ;; - "f") - read -r -p 'file: ' file - sops --encrypt --azure-kv "$kv_key_url" "./secret/$env/$file" > "$encrypted_file_path" - ;; - esac -else - echo "⚠️ Action not allowed." - exit 1 -fi diff --git a/src/domains/gpdingestion-secrets/terraform.sh b/src/domains/gpdingestion-secrets/terraform.sh deleted file mode 100755 index 047a7512d0..0000000000 --- a/src/domains/gpdingestion-secrets/terraform.sh +++ /dev/null @@ -1,324 +0,0 @@ -#!/bin/bash -############################################################ -# Terraform script for managing infrastructure on Azure -# Fingerprint: d2hhdHlvdXdhbnQ/Cg== -############################################################ -# Global variables -# Version format x.y accepted -vers="1.11" -script_name=$(basename "$0") -git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/azure/${script_name}" -tmp_file="${script_name}.new" -# Check if the third parameter exists and is a file -if [ -n "$3" ] && [ -f "$3" ]; then - FILE_ACTION=true -else - FILE_ACTION=false -fi - -# Define functions -function clean_environment() { - rm -rf .terraform - rm tfplan 2>/dev/null - echo "cleaned!" -} - -function download_tool() { - #default value - cpu_type="intel" - os_type=$(uname) - - # only on MacOS - if [ "$os_type" == "Darwin" ]; then - cpu_brand=$(sysctl -n machdep.cpu.brand_string) - if grep -q -i "intel" <<< "$cpu_brand"; then - cpu_type="intel" - else - cpu_type="arm" - fi - fi - - echo $cpu_type - tool=$1 - git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/golang/${tool}_${cpu_type}" - if ! command -v $tool &> /dev/null; then - if ! curl -sL "$git_repo" -o "$tool"; then - echo "Error downloading ${tool}" - return 1 - else - chmod +x $tool - echo "${tool} downloaded! Please note this tool WON'T be copied in your **/bin folder for safety reasons. -You need to do it yourself!" - read -p "Press enter to continue" - - - fi - fi -} - -function extract_resources() { - TF_FILE=$1 - ENV=$2 - TARGETS="" - - # Check if the file exists - if [ ! -f "$TF_FILE" ]; then - echo "File $TF_FILE does not exist." - exit 1 - fi - - # Check if the directory exists - if [ ! -d "./env/$ENV" ]; then - echo "Directory ./env/$ENV does not exist." - exit 1 - fi - - TMP_FILE=$(mktemp) - grep -E '^resource|^module' $TF_FILE > $TMP_FILE - - while read -r line ; do - TYPE=$(echo $line | cut -d '"' -f 1 | tr -d ' ') - if [ "$TYPE" == "module" ]; then - NAME=$(echo $line | cut -d '"' -f 2) - TARGETS+=" -target=\"$TYPE.$NAME\"" - else - NAME1=$(echo $line | cut -d '"' -f 2) - NAME2=$(echo $line | cut -d '"' -f 4) - TARGETS+=" -target=\"$NAME1.$NAME2\"" - fi - done < $TMP_FILE - - rm $TMP_FILE - - echo "./terraform.sh $action $ENV $TARGETS" -} - -function help_usage() { - echo "terraform.sh Version ${vers}" - echo - echo "Usage: ./script.sh [ACTION] [ENV] [OTHER OPTIONS]" - echo "es. ACTION: init, apply, plan, etc." - echo "es. ENV: dev, uat, prod, etc." - echo - echo "Available actions:" - echo " clean Remove .terraform* folders and tfplan files" - echo " help This help" - echo " list List every environment available" - echo " update Update this script if possible" - echo " summ Generate summary of Terraform plan" - echo " tflist Generate an improved output of terraform state list" - echo " tlock Generate or update the dependency lock file" - echo " * any terraform option" -} - -function init_terraform() { - if [ -n "$env" ]; then - terraform init -reconfigure -backend-config="./env/$env/backend.tfvars" - else - echo "ERROR: no env configured!" - exit 1 - fi -} - -function list_env() { - # Check if env directory exists - if [ ! -d "./env" ]; then - echo "No environment directory found" - exit 1 - fi - - # List subdirectories under env directory - env_list=$(ls -d ./env/*/ 2>/dev/null) - - # Check if there are any subdirectories - if [ -z "$env_list" ]; then - echo "No environments found" - exit 1 - fi - - # Print the list of environments - echo "Available environments:" - for env in $env_list; do - env_name=$(echo "$env" | sed 's#./env/##;s#/##') - echo "- $env_name" - done -} - -function other_actions() { - if [ -n "$env" ] && [ -n "$action" ]; then - terraform "$action" -var-file="./env/$env/terraform.tfvars" -compact-warnings $other - else - echo "ERROR: no env or action configured!" - exit 1 - fi -} - -function state_output_taint_actions() { - if [ "$action" == "tflist" ]; then - # If 'tflist' is not installed globally and there is no 'tflist' file in the current directory, - # attempt to download the 'tflist' tool - if ! command -v tflist &> /dev/null && [ ! -f "tflist" ]; then - download_tool "tflist" - if [ $? -ne 0 ]; then - echo "Error: Failed to download tflist!!" - exit 1 - else - echo "tflist downloaded!" - fi - fi - if command -v tflist &> /dev/null; then - terraform state list | tflist - else - terraform state list | ./tflist - fi - else - terraform $action $other - fi -} - - -function parse_tfplan_option() { - # Create an array to contain arguments that do not start with '-tfplan=' - local other_args=() - - # Loop over all arguments - for arg in "$@"; do - # If the argument starts with '-tfplan=', extract the file name - if [[ "$arg" =~ ^-tfplan= ]]; then - echo "${arg#*=}" - else - # If the argument does not start with '-tfplan=', add it to the other_args array - other_args+=("$arg") - fi - done - - # Print all arguments in other_args separated by spaces - echo "${other_args[@]}" -} - -function tfsummary() { - local plan_file - plan_file=$(parse_tfplan_option "$@") - if [ -z "$plan_file" ]; then - plan_file="tfplan" - fi - action="plan" - other="-out=${plan_file}" - other_actions - if [ -n "$(command -v tf-summarize)" ]; then - tf-summarize -tree "${plan_file}" - else - echo "tf-summarize is not installed" - fi - if [ "$plan_file" == "tfplan" ]; then - rm $plan_file - fi -} - -function update_script() { - # Check if the repository was cloned successfully - if ! curl -sL "$git_repo" -o "$tmp_file"; then - echo "Error cloning the repository" - rm "$tmp_file" 2>/dev/null - return 1 - fi - - # Check if a newer version exists - remote_vers=$(sed -n '8s/vers="\(.*\)"/\1/p' "$tmp_file") - if [ "$(printf '%s\n' "$vers" "$remote_vers" | sort -V | tail -n 1)" == "$vers" ]; then - echo "The local script version is equal to or newer than the remote version." - rm "$tmp_file" 2>/dev/null - return 0 - fi - - # Check the fingerprint - local_fingerprint=$(sed -n '4p' "$0") - remote_fingerprint=$(sed -n '4p' "$tmp_file") - - if [ "$local_fingerprint" != "$remote_fingerprint" ]; then - echo "The local and remote file fingerprints do not match." - rm "$tmp_file" 2>/dev/null - return 0 - fi - - # Show the current and available versions to the user - echo "Current script version: $vers" - echo "Available script version: $remote_vers" - - # Ask the user if they want to update the script - read -rp "Do you want to update the script to version $remote_vers? (y/n): " answer - - if [ "$answer" == "y" ] || [ "$answer" == "Y" ]; then - # Replace the local script with the updated version - cp "$tmp_file" "$script_name" - chmod +x "$script_name" - rm "$tmp_file" 2>/dev/null - - echo "Script successfully updated to version $remote_vers" - else - echo "Update canceled by the user" - fi - - rm "$tmp_file" 2>/dev/null -} - -# Check arguments number -if [ "$#" -lt 1 ]; then - help_usage - exit 0 -fi - -# Parse arguments -action=$1 -env=$2 -filetf=$3 -shift 2 -other=$@ - -if [ -n "$env" ]; then - # shellcheck source=/dev/null - source "./env/$env/backend.ini" - if [ -z "$(command -v az)" ]; then - echo "az not found, cannot proceed" - exit 1 - fi - az account set -s "${subscription}" -fi - -# Call appropriate function based on action -case $action in - clean) - clean_environment - ;; - ?|help|-h) - help_usage - ;; - init) - init_terraform "$other" - ;; - list) - list_env - ;; - output|state|taint|tflist) - init_terraform - state_output_taint_actions $other - ;; - summ) - init_terraform - tfsummary "$other" - ;; - tlock) - terraform providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=darwin_arm64 -platform=linux_amd64 - ;; - update) - update_script - ;; - *) - if [ "$FILE_ACTION" = true ]; then - extract_resources "$filetf" "$env" - else - init_terraform - other_actions "$other" - fi - ;; -esac diff --git a/src/domains/gpdingestion-secrets/terrasops.sh b/src/domains/gpdingestion-secrets/terrasops.sh deleted file mode 100644 index 32be3bd04f..0000000000 --- a/src/domains/gpdingestion-secrets/terrasops.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# set -x # Uncomment this line to enable debug mode - -# -# ℹ️ This script is used by terraform, to decrypt all secrets on sops and export them to json. -# This way it can loop through them and use them to insert them inside the KV -# ⚠️ Do not add additional echos to the script in case of golden path, -# as the script only needs to return a json -# - -eval "$(jq -r '@sh "export terrasops_env=\(.env)"')" - -# shellcheck disable=SC1090 -source "./secret/$terrasops_env/secret.ini" -encrypted_file_path="./secret/$terrasops_env/$file_crypted" - -if [ -f "$encrypted_file_path" ]; then - # Load the values of azure_kv.vault_url and azure_kv.name from the JSON file - azure_kv_vault_url=$(jq -r '.sops.azure_kv[0].vault_url' "$encrypted_file_path") - azure_kv_name=$(jq -r '.sops.azure_kv[0].name' "$encrypted_file_path") - - if [ -z "$azure_kv_vault_url" ] || [ -z "$azure_kv_name" ]; then - echo "❌ Error: Unable to load the values of azure_kv.vault_url and azure_kv.name from the JSON file" >&2 - exit 1 - fi - sops -d --azure-kv "azure_kv_vault_url" "$encrypted_file_path" | jq -c -else - echo "{}" | jq -c -fi From 923b306458783dcfe8f2df619dde80735ce2dba6 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 28 Oct 2024 18:13:45 +0100 Subject: [PATCH 08/55] fix --- .../gpdingestion-app/.terraform.lock.hcl | 102 ------ src/domains/gpdingestion-app/00_data.tf | 17 - src/domains/gpdingestion-app/00_keyvault.tf | 10 - src/domains/gpdingestion-app/00_monitor.tf | 35 -- src/domains/gpdingestion-app/00_network.tf | 15 - src/domains/gpdingestion-app/01_network.tf | 9 - src/domains/gpdingestion-app/02_namespace.tf | 28 -- .../03_serviceaccounts_azure_devops.tf | 67 ---- .../05_aks_middleware_tools.tf | 49 --- src/domains/gpdingestion-app/06_keyvault.tf | 38 -- src/domains/gpdingestion-app/90_pdb.tf | 15 - src/domains/gpdingestion-app/99_locals.tf | 43 --- src/domains/gpdingestion-app/99_main.tf | 53 --- src/domains/gpdingestion-app/99_variables.tf | 253 -------------- src/domains/gpdingestion-app/README.md | 106 ------ .../gpdingestion-app/env/itn-dev/backend.ini | 1 - .../env/itn-dev/backend.tfvars | 4 - .../env/itn-dev/terraform.tfvars | 55 --- .../gpdingestion-app/env/itn-prod/backend.ini | 1 - .../env/itn-prod/backend.tfvars | 4 - .../env/itn-prod/terraform.tfvars | 64 ---- .../gpdingestion-app/env/itn-uat/backend.ini | 1 - .../env/itn-uat/backend.tfvars | 4 - .../env/itn-uat/terraform.tfvars | 55 --- .../helm/cert-mounter.yaml.tpl | 13 - src/domains/gpdingestion-app/terraform.sh | 324 ------------------ .../05_debezium_connect.tf | 36 +- src/domains/gps-app/99_variables.tf | 97 ++++++ .../gps-app/env/weu-dev/terraform.tfvars | 18 + .../set_registry_secrets.sh | 12 +- .../yaml/debezium-rbac.yaml | 0 .../yaml/debezium-role.yaml | 0 .../yaml/debezium-secrets.yaml | 0 .../yaml/kafka-connect.yaml | 1 + .../yaml/postgres-connector.yaml | 26 +- .../yaml/zookeeper.yaml | 0 36 files changed, 156 insertions(+), 1400 deletions(-) delete mode 100644 src/domains/gpdingestion-app/.terraform.lock.hcl delete mode 100644 src/domains/gpdingestion-app/00_data.tf delete mode 100644 src/domains/gpdingestion-app/00_keyvault.tf delete mode 100644 src/domains/gpdingestion-app/00_monitor.tf delete mode 100644 src/domains/gpdingestion-app/00_network.tf delete mode 100644 src/domains/gpdingestion-app/01_network.tf delete mode 100644 src/domains/gpdingestion-app/02_namespace.tf delete mode 100644 src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf delete mode 100644 src/domains/gpdingestion-app/05_aks_middleware_tools.tf delete mode 100644 src/domains/gpdingestion-app/06_keyvault.tf delete mode 100644 src/domains/gpdingestion-app/90_pdb.tf delete mode 100644 src/domains/gpdingestion-app/99_locals.tf delete mode 100644 src/domains/gpdingestion-app/99_main.tf delete mode 100644 src/domains/gpdingestion-app/99_variables.tf delete mode 100644 src/domains/gpdingestion-app/README.md delete mode 100644 src/domains/gpdingestion-app/env/itn-dev/backend.ini delete mode 100644 src/domains/gpdingestion-app/env/itn-dev/backend.tfvars delete mode 100644 src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars delete mode 100644 src/domains/gpdingestion-app/env/itn-prod/backend.ini delete mode 100644 src/domains/gpdingestion-app/env/itn-prod/backend.tfvars delete mode 100644 src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars delete mode 100644 src/domains/gpdingestion-app/env/itn-uat/backend.ini delete mode 100644 src/domains/gpdingestion-app/env/itn-uat/backend.tfvars delete mode 100644 src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars delete mode 100644 src/domains/gpdingestion-app/helm/cert-mounter.yaml.tpl delete mode 100755 src/domains/gpdingestion-app/terraform.sh rename src/domains/{gpdingestion-app => gps-app}/05_debezium_connect.tf (66%) rename src/domains/{gpdingestion-app => gps-app}/set_registry_secrets.sh (66%) rename src/domains/{gpdingestion-app => gps-app}/yaml/debezium-rbac.yaml (100%) rename src/domains/{gpdingestion-app => gps-app}/yaml/debezium-role.yaml (100%) rename src/domains/{gpdingestion-app => gps-app}/yaml/debezium-secrets.yaml (100%) rename src/domains/{gpdingestion-app => gps-app}/yaml/kafka-connect.yaml (98%) rename src/domains/{gpdingestion-app => gps-app}/yaml/postgres-connector.yaml (68%) rename src/domains/{gpdingestion-app => gps-app}/yaml/zookeeper.yaml (100%) diff --git a/src/domains/gpdingestion-app/.terraform.lock.hcl b/src/domains/gpdingestion-app/.terraform.lock.hcl deleted file mode 100644 index 97cef8e563..0000000000 --- a/src/domains/gpdingestion-app/.terraform.lock.hcl +++ /dev/null @@ -1,102 +0,0 @@ -# This file is maintained automatically by "terraform init". -# Manual edits may be lost in future updates. - -provider "registry.terraform.io/hashicorp/azuread" { - version = "2.47.0" - constraints = "<= 2.47.0" - hashes = [ - "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", - "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", - "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", - "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", - "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", - "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", - "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", - "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", - "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", - "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", - "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", - "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", - "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", - ] -} - -provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.97.1" - constraints = ">= 3.30.0, ~> 3.30, <= 3.97.1, <= 3.106.0" - hashes = [ - "h1:LtwGbd4HEb5QCXmdxSvTjPSh8/Gp8eAQMYfiAKaubV4=", - "zh:15171efcc3aa3a37748c502c493cb16ecff603b81ada4499a843574976bac524", - "zh:2ca6c13a4a96f67763ecced0015c7b101ee02d54ea54b28a8df4ae06468071b1", - "zh:2e3c77dbfd8f760132ecef2d6117e939cbea26b96aba5e4d926e7f7f0f7afe72", - "zh:4bc346eece1622be93c73801d8256502b11fd7c2e7f7cea12d048bb9fc9fe900", - "zh:4f1042942ed8d0433680a367527289459d43b0894a51eaba83ac414e80d5187f", - "zh:63e674c31482ae3579ea84daf5b1ba066ce40cb23475f54e17b6b131320a1bec", - "zh:8327148766dcb7a174673729a832c8095d7e137d0e6c7e2a9a01da48b8b73fbe", - "zh:851b3ae417059a80c7813e7f0063298a590a42f056004f2c2558ea14061c207e", - "zh:ac081b48907139c121a422ae9b1f40fc72c6aaaeb05cbdbf848102a6a5f426f4", - "zh:dc1d663df2d95e4ba91070ceb20d3560b6ea5c465d39c57a5979319302643e41", - "zh:ed26457367cbbb94237e935d297cb31b5687f9abf697377da0ee46974480db9b", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/helm" { - version = "2.12.1" - constraints = "~> 2.12, <= 2.12.1" - hashes = [ - "h1:7wfYOAeSEchHB8idNl+2jf+OkFi9zFSOLWkEZFuTCik=", - "zh:1d623fb1662703f2feb7860e3c795d849c77640eecbc5a776784d08807b15004", - "zh:253a5bc62ba2c4314875139e3fbd2feaad5ef6b0fb420302a474ab49e8e51a38", - "zh:282358f4ad4f20d0ccaab670b8645228bfad1c03ac0d0df5889f0aea8aeac01a", - "zh:4fd06af3091a382b3f0d8f0a60880f59640d2b6d9d6a31f9a873c6f1bde1ec50", - "zh:6816976b1830f5629ae279569175e88b497abbbac30ee809948a1f923c67a80d", - "zh:7d82c4150cdbf48cfeec867be94c7b9bd7682474d4df0ebb7e24e148f964844f", - "zh:83f062049eea2513118a4c6054fb06c8600bac96196f25aed2cc21898ec86e93", - "zh:a79eec0cf4c08fca79e44033ec6e470f25ff23c3e2c7f9bc707ed7771c1072c0", - "zh:b2b2d904b2821a6e579910320605bc478bbef063579a23fbfdd6fcb5871b81f8", - "zh:e91177ca06a15487fc570cb81ecef6359aa399459ea2aa7c4f7367ba86f6fcad", - "zh:e976bcb82996fc4968f8382bbcb6673efb1f586bf92074058a232028d97825b1", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/kubernetes" { - version = "2.29.0" - constraints = "~> 2.27, <= 2.29.0" - hashes = [ - "h1:Igs0JTtmzn5q7RHqrvrTMCD/DCSLPMinvUnhYZ2oITw=", - "zh:3edd5dc319b95fe94e61b82d10c1ce7fb53a2f21b067ddb742f2d7d0d19dd113", - "zh:4b9096e6d0cfa0efd4c89270e3d25fea49db570e2cfbe49c5d1de085a15f2578", - "zh:5397573838bcb8844248c8d6ac93cca7f39a0b707ac3ce7a7b306c50c261c195", - "zh:5d635370720d356b7bcb5756ca28de3275ca32ca1ef0201414caecd3a14759ac", - "zh:71a52280408f3fb0ff1866a9ab8059b0d9bde5481869658798e0773461f22eff", - "zh:748663ef0248d2d95f5dea2974332432a395165657856878c5dc6f000b37cc25", - "zh:7fbc1e084bbbb51e31afd3df0c77e833ae59e88cf42b9e2c17b0b1a1e3894723", - "zh:ae89b4be473b446270fa24dc1ef51b0cc4c2a528d9838ec15246d28bac165df3", - "zh:b6433970d680a0cc9898f915224508b5ece86ae4418372fa6bebd2a9d344f226", - "zh:bf871955cf49015e6a0433e814a22a109c1537a775b8b5dc7b37ad05c324904a", - "zh:c16fac91b2197b443a191d98cf37424feed550387ab11bd1427bde819722005e", - "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - ] -} - -provider "registry.terraform.io/hashicorp/null" { - version = "3.2.1" - constraints = "~> 3.2, <= 3.2.1" - hashes = [ - "h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=", - "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", - "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", - "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", - "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", - "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", - "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", - "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", - "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", - "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", - "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", - ] -} diff --git a/src/domains/gpdingestion-app/00_data.tf b/src/domains/gpdingestion-app/00_data.tf deleted file mode 100644 index fc13f587b2..0000000000 --- a/src/domains/gpdingestion-app/00_data.tf +++ /dev/null @@ -1,17 +0,0 @@ -### EVH -data "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" { - name = "cdc-connection-string" - namespace_name = "${local.project}-evh" - resource_group_name = "${local.project}-evh-rg" -} - -data "azurerm_eventhub_namespace" "eventhub" { - name = "${local.project}-evh" - namespace_name = "${local.project}-evh" - resource_group_name = "${local.project}-evh-rg" -} - -data "azurerm_postgresql_database" "apd_db" { - name = var.postgres_db_name - resource_group_name = "${local.project}-evh-rg" -} diff --git a/src/domains/gpdingestion-app/00_keyvault.tf b/src/domains/gpdingestion-app/00_keyvault.tf deleted file mode 100644 index c94a899cca..0000000000 --- a/src/domains/gpdingestion-app/00_keyvault.tf +++ /dev/null @@ -1,10 +0,0 @@ -data "azurerm_key_vault" "kv" { - name = "${local.project}-kv" - resource_group_name = "${local.project}-sec-rg" -} - - -data "azurerm_kubernetes_cluster" "aks" { - name = local.aks_name - resource_group_name = local.aks_resource_group_name -} diff --git a/src/domains/gpdingestion-app/00_monitor.tf b/src/domains/gpdingestion-app/00_monitor.tf deleted file mode 100644 index 311dc4ff7d..0000000000 --- a/src/domains/gpdingestion-app/00_monitor.tf +++ /dev/null @@ -1,35 +0,0 @@ -# -# 🇮🇹 Monitor Italy -# -data "azurerm_resource_group" "monitor_italy_rg" { - name = var.monitor_italy_resource_group_name -} - -data "azurerm_log_analytics_workspace" "log_analytics_italy" { - name = var.log_analytics_italy_workspace_name - resource_group_name = var.log_analytics_italy_workspace_resource_group_name -} - -data "azurerm_application_insights" "application_insights_italy" { - name = local.monitor_appinsights_italy_name - resource_group_name = data.azurerm_resource_group.monitor_italy_rg.name -} - -# -# Actions Group -# -data "azurerm_monitor_action_group" "slack" { - name = local.monitor_action_group_slack_name - resource_group_name = var.monitor_italy_resource_group_name -} - -data "azurerm_monitor_action_group" "email" { - resource_group_name = var.monitor_italy_resource_group_name - name = local.monitor_action_group_email_name -} - -data "azurerm_monitor_action_group" "opsgenie" { - count = var.env_short == "p" ? 1 : 0 - resource_group_name = var.monitor_resource_group_name - name = local.monitor_action_group_opsgenie_name -} diff --git a/src/domains/gpdingestion-app/00_network.tf b/src/domains/gpdingestion-app/00_network.tf deleted file mode 100644 index 355c8e2333..0000000000 --- a/src/domains/gpdingestion-app/00_network.tf +++ /dev/null @@ -1,15 +0,0 @@ -data "azurerm_virtual_network" "vnet" { - name = local.vnet_name - resource_group_name = local.vnet_resource_group_name -} - -data "azurerm_private_dns_zone" "internal" { - name = local.internal_dns_zone_name - resource_group_name = local.internal_dns_zone_resource_group_name -} - -data "azurerm_subnet" "apim_vnet" { - name = local.pagopa_apim_snet - resource_group_name = local.pagopa_vnet_rg - virtual_network_name = local.pagopa_vnet_integration -} diff --git a/src/domains/gpdingestion-app/01_network.tf b/src/domains/gpdingestion-app/01_network.tf deleted file mode 100644 index 73614770ca..0000000000 --- a/src/domains/gpdingestion-app/01_network.tf +++ /dev/null @@ -1,9 +0,0 @@ -#-------------------------------------------------- - -resource "azurerm_private_dns_a_record" "ingress" { - name = local.ingress_hostname - zone_name = data.azurerm_private_dns_zone.internal.name - resource_group_name = local.internal_dns_zone_resource_group_name - ttl = 3600 - records = [var.ingress_load_balancer_ip] -} diff --git a/src/domains/gpdingestion-app/02_namespace.tf b/src/domains/gpdingestion-app/02_namespace.tf deleted file mode 100644 index 759fa6152b..0000000000 --- a/src/domains/gpdingestion-app/02_namespace.tf +++ /dev/null @@ -1,28 +0,0 @@ -resource "kubernetes_namespace" "namespace" { - metadata { - name = var.domain - } -} - -module "workload_identity" { - source = "./.terraform/modules/__v3__/kubernetes_workload_identity_init" - - workload_identity_name_prefix = "${var.domain}-workload-identity" - workload_identity_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name - workload_identity_location = var.location -} - -module "workload_identity" { - source = "./.terraform/modules/__v3__/kubernetes_workload_identity_configuration" - - workload_identity_name_prefix = "${var.domain}-poc" - workload_identity_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name - aks_name = data.azurerm_kubernetes_cluster.aks.name - aks_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name - namespace = var.domain - - key_vault_id = data.azurerm_key_vault.kv.id - key_vault_certificate_permissions = ["Get"] - key_vault_key_permissions = ["Get"] - key_vault_secret_permissions = ["Get"] -} diff --git a/src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf b/src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf deleted file mode 100644 index 0b297b7fdf..0000000000 --- a/src/domains/gpdingestion-app/03_serviceaccounts_azure_devops.tf +++ /dev/null @@ -1,67 +0,0 @@ -resource "kubernetes_namespace" "namespace_system" { - metadata { - name = "${var.domain}-system" - } -} - -module "kubernetes_service_account" { - source = "./.terraform/modules/__v3__/kubernetes_service_account?ref=v8.18.0" - name = "azure-devops" - namespace = "${var.domain}-system" -} - -#tfsec:ignore:AZU023 -resource "azurerm_key_vault_secret" "azure_devops_sa_token" { - depends_on = [module.kubernetes_service_account] - name = "${local.aks_name}-azure-devops-sa-token" - value = module.kubernetes_service_account.sa_token # base64 value - content_type = "text/plain" - - key_vault_id = data.azurerm_key_vault.kv.id -} - -#tfsec:ignore:AZU023 -resource "azurerm_key_vault_secret" "azure_devops_sa_cacrt" { - depends_on = [module.kubernetes_service_account] - name = "${local.aks_name}-azure-devops-sa-cacrt" - value = module.kubernetes_service_account.sa_ca_cert # base64 value - content_type = "text/plain" - - key_vault_id = data.azurerm_key_vault.kv.id -} - -#-------------------------------------------------------------------------------------------------- - -resource "kubernetes_role_binding" "deployer_binding" { - metadata { - name = "deployer-binding" - namespace = kubernetes_namespace.namespace.metadata[0].name - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "ClusterRole" - name = "cluster-deployer" - } - subject { - kind = "ServiceAccount" - name = "azure-devops" - namespace = kubernetes_namespace.namespace_system.metadata[0].name - } -} - -resource "kubernetes_role_binding" "system_deployer_binding" { - metadata { - name = "system-deployer-binding" - namespace = kubernetes_namespace.namespace_system.metadata[0].name - } - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "ClusterRole" - name = "system-cluster-deployer" - } - subject { - kind = "ServiceAccount" - name = "azure-devops" - namespace = kubernetes_namespace.namespace_system.metadata[0].name - } -} diff --git a/src/domains/gpdingestion-app/05_aks_middleware_tools.tf b/src/domains/gpdingestion-app/05_aks_middleware_tools.tf deleted file mode 100644 index 83a33add1e..0000000000 --- a/src/domains/gpdingestion-app/05_aks_middleware_tools.tf +++ /dev/null @@ -1,49 +0,0 @@ -module "tls_checker" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//tls_checker?ref=v8.22.0" - - https_endpoint = local.domain_hostname - alert_name = local.domain_hostname - alert_enabled = true - helm_chart_present = true - namespace = kubernetes_namespace.namespace.metadata[0].name - location_string = var.location_string - kv_secret_name_for_application_insights_connection_string = "app-insight-connection-string" - application_insights_resource_group = data.azurerm_resource_group.monitor_italy_rg.name - application_insights_id = data.azurerm_application_insights.application_insights_italy.id - application_insights_action_group_ids = [data.azurerm_monitor_action_group.slack.id, data.azurerm_monitor_action_group.email.id] - keyvault_name = data.azurerm_key_vault.kv.name - keyvault_tenant_id = data.azurerm_client_config.current.tenant_id -} - -resource "helm_release" "cert_mounter" { - name = "cert-mounter-blueprint" - repository = "https://pagopa.github.io/aks-helm-cert-mounter-blueprint" - chart = "cert-mounter-blueprint" - version = "1.0.4" - namespace = var.domain - timeout = 120 - force_update = true - - values = [ - templatefile("${path.root}/helm/cert-mounter.yaml.tpl", { - NAMESPACE = var.domain, - DOMAIN = var.domain, - CERTIFICATE_NAME = replace(local.domain_hostname, ".", "-"), - ENV_SHORT = var.env_short, - KV_NAME = data.azurerm_key_vault.kv.name - }) - ] -} - -resource "helm_release" "reloader" { - name = "reloader" - repository = "https://stakater.github.io/stakater-charts" - chart = "reloader" - version = "v1.0.69" - namespace = kubernetes_namespace.namespace.metadata[0].name - - set { - name = "reloader.watchGlobally" - value = "false" - } -} diff --git a/src/domains/gpdingestion-app/06_keyvault.tf b/src/domains/gpdingestion-app/06_keyvault.tf deleted file mode 100644 index 152ae794da..0000000000 --- a/src/domains/gpdingestion-app/06_keyvault.tf +++ /dev/null @@ -1,38 +0,0 @@ -locals { - aks_api_url = var.env_short == "d" ? data.azurerm_kubernetes_cluster.aks.fqdn : data.azurerm_kubernetes_cluster.aks.private_fqdn -} - -#tfsec:ignore:AZU023 -resource "azurerm_key_vault_secret" "aks_apiserver_url" { - name = "${local.aks_name}-apiserver-url" - value = "https://${local.aks_api_url}:443" - content_type = "text/plain" - - key_vault_id = data.azurerm_key_vault.kv.id -} - -## Manual secrets - -resource "azurerm_key_vault_secret" "application_insights_connection_string" { - name = "app-insight-connection-string" - value = data.azurerm_application_insights.application_insights_italy.connection_string - content_type = "text/plain" - key_vault_id = data.azurerm_key_vault.kv.id -} - - -resource "azurerm_key_vault_secret" "tenant_id" { - name = "tenant-id" - value = data.azurerm_subscription.current.tenant_id - content_type = "text/plain" - key_vault_id = data.azurerm_key_vault.kv.id -} - -# Event Hub - -resource "azurerm_key_vault_secret" "ehub_gpd_ingestion_jaas_config" { - name = "ehub-${var.env_short}-gpd-ingestion-jaas-config" - value = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${azurerm_eventhub_authorization_rule.cdc_connection_string.primary_connection_string}\";" - content_type = "text/plain" - key_vault_id = data.azurerm_key_vault.kv.id -} diff --git a/src/domains/gpdingestion-app/90_pdb.tf b/src/domains/gpdingestion-app/90_pdb.tf deleted file mode 100644 index 68c5c276a6..0000000000 --- a/src/domains/gpdingestion-app/90_pdb.tf +++ /dev/null @@ -1,15 +0,0 @@ -resource "kubernetes_pod_disruption_budget_v1" "gpd_ingestion" { - - for_each = var.pod_disruption_budgets - - metadata { - namespace = kubernetes_namespace.namespace.metadata[0].name - name = each.key - } - spec { - min_available = each.value.minAvailable - selector { - match_labels = each.value.matchLabels - } - } -} diff --git a/src/domains/gpdingestion-app/99_locals.tf b/src/domains/gpdingestion-app/99_locals.tf deleted file mode 100644 index 9b8dffcb17..0000000000 --- a/src/domains/gpdingestion-app/99_locals.tf +++ /dev/null @@ -1,43 +0,0 @@ -locals { - product = "${var.prefix}-${var.env_short}" - project_short = "${var.prefix}-${var.env_short}-${var.domain}" - project = "${var.prefix}-${var.env_short}-${var.location_short}-${var.domain}" - - location_short_weu = "weu" - project_short_weu = "${var.prefix}-${var.env_short}-${local.location_short_weu}" - - project_core_itn = "${var.prefix}-${var.env_short}-${var.location_short}-core" - - - monitor_action_group_slack_name = "SlackPagoPA" - monitor_action_group_email_name = "PagoPA" - monitor_action_group_opsgenie_name = "Opsgenie" - monitor_appinsights_name = "${local.product}-appinsights" - monitor_appinsights_italy_name = "${local.project_core_itn}-appinsights" - - vnet_name = "${var.prefix}-${var.env_short}-${var.location_short}-vnet" - vnet_resource_group_name = "${var.prefix}-${var.env_short}-${var.location_short}-vnet-rg" - - aks_name = "${local.product}-${var.location_short}-${var.instance}-aks" - aks_resource_group_name = "${local.product}-${var.location_short}-${var.instance}-aks-rg" - - ingress_hostname = "${var.domain}.itn" - internal_dns_zone_name = "${var.dns_zone_internal_prefix}.${var.external_domain}" - internal_dns_zone_resource_group_name = "${local.product}-vnet-rg" - - pagopa_apim_snet = "${local.product}-apim-snet" - pagopa_vnet_integration = "pagopa-${var.env_short}-vnet-integration" - pagopa_vnet_rg = "pagopa-${var.env_short}-vnet-rg" - - domain_hostname = "${var.dns_zone_prefix}.${local.internal_dns_zone_name}" - - pagopa_apim_name = "${local.product}-apim" - pagopa_apim_rg = "${local.product}-api-rg" - - apim_hostname = "api.${var.apim_dns_zone_prefix}.${var.external_domain}" - hostname = var.env == "prod" ? "${var.domain}.itn.internal.platform.pagopa.it" : "${var.domain}.itn.internal.${var.env}.platform.pagopa.it" - - - evt_hub_location = "${local.location_short_weu}-core" - -} diff --git a/src/domains/gpdingestion-app/99_main.tf b/src/domains/gpdingestion-app/99_main.tf deleted file mode 100644 index 3ac065d2cd..0000000000 --- a/src/domains/gpdingestion-app/99_main.tf +++ /dev/null @@ -1,53 +0,0 @@ -terraform { - required_version = ">= 1.6.0" - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "<= 3.116.0" - } - azuread = { - source = "hashicorp/azuread" - version = "<= 3.0.2" - } - null = { - source = "hashicorp/null" - version = "<= 3.2.3" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = "<= 2.33.0" - } - helm = { - source = "hashicorp/helm" - version = "<= 2.16.0" - } - } - - backend "azurerm" {} -} - -provider "azurerm" { - features { - key_vault { - purge_soft_delete_on_destroy = false - } - } -} - -data "azurerm_subscription" "current" {} - -data "azurerm_client_config" "current" {} - -provider "kubernetes" { - config_path = "${var.k8s_kube_config_path_prefix}/config-${local.aks_name}" -} - -provider "helm" { - kubernetes { - config_path = "${var.k8s_kube_config_path_prefix}/config-${local.aks_name}" - } -} - -module "__v3__" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" -} diff --git a/src/domains/gpdingestion-app/99_variables.tf b/src/domains/gpdingestion-app/99_variables.tf deleted file mode 100644 index 410c72ea94..0000000000 --- a/src/domains/gpdingestion-app/99_variables.tf +++ /dev/null @@ -1,253 +0,0 @@ -# general - -variable "prefix" { - type = string - validation { - condition = ( - length(var.prefix) <= 6 - ) - error_message = "Max length is 6 chars." - } -} - -variable "env" { - type = string -} - -variable "env_short" { - type = string - validation { - condition = ( - length(var.env_short) == 1 - ) - error_message = "Length must be 1 chars." - } -} - -variable "domain" { - type = string - validation { - condition = ( - length(var.domain) <= 12 - ) - error_message = "Max length is 12 chars." - } -} - -variable "location" { - type = string - description = "One of westeurope, northeurope" -} - -variable "location_short" { - type = string - validation { - condition = ( - length(var.location_short) == 3 - ) - error_message = "Length must be 3 chars." - } - description = "One of wue, neu" -} - -variable "location_string" { - type = string - description = "One of West Europe, North Europe" -} - -variable "instance" { - type = string - description = "One of beta, prod01, prod02" -} - -variable "tags" { - type = map(any) - default = { - CreatedBy = "Terraform" - } -} - -### Features flags - -variable "is_feature_enabled" { - type = object({ - gpdingestion = bool - }) - default = { - gpdingestion = false - } -} -### External resources - -variable "monitor_resource_group_name" { - type = string - description = "Monitor resource group name" -} - -variable "log_analytics_workspace_name" { - type = string - description = "Specifies the name of the Log Analytics Workspace." -} - -variable "log_analytics_workspace_resource_group_name" { - type = string - description = "The name of the resource group in which the Log Analytics workspace is located in." -} - -variable "monitor_italy_resource_group_name" { - type = string - description = "Monitor Italy resource group name" -} - -variable "log_analytics_italy_workspace_name" { - type = string - description = "Specifies the name of the Log Analytics Workspace Italy." -} - -variable "log_analytics_italy_workspace_resource_group_name" { - type = string - description = "The name of the resource group in which the Log Analytics workspace Italy is located in." -} - - -### Aks -variable "ingress_load_balancer_ip" { - type = string -} - -variable "k8s_kube_config_path_prefix" { - type = string - default = "~/.kube" -} - -variable "external_domain" { - type = string - default = null - description = "Domain for delegation" -} - -variable "dns_zone_internal_prefix" { - type = string - default = null - description = "The dns subdomain." -} - -variable "apim_dns_zone_prefix" { - type = string - default = null - description = "The dns subdomain for apim." -} - -# DNS - -variable "dns_zone_prefix" { - type = string - default = null - description = "The wallet dns subdomain." -} - -### PDB -variable "pod_disruption_budgets" { - type = map(object({ - name = optional(string, null) - minAvailable = optional(number, null) - matchLabels = optional(map(any), {}) - })) - description = "Pod disruption budget for domain namespace" - default = {} -} - -variable "zookeeper_replicas" { - type = number - description = "Zookeeper Replicas" - default = 1 -} - -variable "zookeeper_request_memory" { - type = string - description = "Zookeeper Request Memory" - default = "512m" -} - -variable "zookeeper_request_cpu" { - type = string - description = "Zookeeper Request CPU" - default = "0.5" -} - -variable "zookeeper_limits_memory" { - type = string - description = "Zookeeper Limit Memory" - default = "512mi" -} - -variable "zookeeper_limits_cpu" { - type = string - description = "Zookeeper Limit CPU" - default = "0.5" -} - -variable "zookeeper_jvm_xms" { - type = string - description = "Zookeeper Jvm Xms" - default = "512mi" -} - -variable "zookeeper_jvm_xmx" { - type = string - description = "Zookeeper Jvm Xmx" - default = "512mi" -} - -variable "zookeeper_storage_size" { - type = string - description = "Zookeeper Storage Size" - default = "100Gi" -} - -variable "container_registry" { - type = string - description = "Container Registry" -} - -variable "postgres_db_name" { - type = string - description = "Postgres Database Name" - default = "apd" -} - -variable "tasks_max" { - type = string - description = "Number of tasks" - default = "1" -} - -variable "replicas" { - type = number - description = "Number of replicas in cluster" - default = 1 -} - -variable "request_memory" { - type = string - description = "Connect Request Memory" - default = "512m" -} - -variable "request_cpu" { - type = string - description = "Connect Request CPU" - default = "0.5" -} - -variable "limits_memory" { - type = string - description = "Connect Limit Memory" - default = "512mi" -} - -variable "limits_cpu" { - type = string - description = "Connect Limit CPU" - default = "0.5" -} diff --git a/src/domains/gpdingestion-app/README.md b/src/domains/gpdingestion-app/README.md deleted file mode 100644 index 51e2d2e81e..0000000000 --- a/src/domains/gpdingestion-app/README.md +++ /dev/null @@ -1,106 +0,0 @@ -# paymentoptions-app - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.6.0 | -| [azuread](#requirement\_azuread) | <= 2.47.0 | -| [azurerm](#requirement\_azurerm) | <= 3.106.0 | -| [helm](#requirement\_helm) | <= 2.12.1 | -| [kubernetes](#requirement\_kubernetes) | <= 2.29.0 | -| [null](#requirement\_null) | <= 3.2.1 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [apim\_api\_pay\_opt\_mock\_api](#module\_apim\_api\_pay\_opt\_mock\_api) | git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_api | v8.18.0 | -| [apim\_payment\_options\_mock\_product](#module\_apim\_payment\_options\_mock\_product) | git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_product | v8.18.0 | -| [apim\_payment\_options\_product](#module\_apim\_payment\_options\_product) | git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_product | v8.18.0 | -| [kubernetes\_service\_account](#module\_kubernetes\_service\_account) | git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_service_account | v8.18.0 | -| [pod\_identity](#module\_pod\_identity) | git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_pod_identity | v8.18.0 | -| [tls\_checker](#module\_tls\_checker) | git::https://github.com/pagopa/terraform-azurerm-v3.git//tls_checker | v8.22.0 | - -## Resources - -| Name | Type | -|------|------| -| [azurerm_api_management_api_version_set.payment_options_mock_api](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/api_management_api_version_set) | resource | -| [azurerm_api_management_subscription.api_config_subkey](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/api_management_subscription) | resource | -| [azurerm_api_management_subscription.forwarder_subkey](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/api_management_subscription) | resource | -| [azurerm_api_management_subscription.service_payment_options_subkey](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/api_management_subscription) | resource | -| [azurerm_key_vault_secret.aks_apiserver_url](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_key_vault_secret.api_config_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_key_vault_secret.application_insights_connection_string](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_key_vault_secret.azure_devops_sa_cacrt](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_key_vault_secret.azure_devops_sa_token](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_key_vault_secret.ehub_nodo-dei-pagamenti-verify-ko_jaas_config](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_key_vault_secret.ehub_nodo_pagamenti_cache_jaas_config](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_key_vault_secret.ehub_payment-options-re_jaas_config](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_key_vault_secret.forwarder_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_key_vault_secret.service_payment_options_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_key_vault_secret.tenant_id](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | -| [azurerm_monitor_scheduled_query_rules_alert.pagopa-payment-options-rest-availability-upd](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_scheduled_query_rules_alert) | resource | -| [azurerm_monitor_scheduled_query_rules_alert.pagopa-payment-options-service-responsetime-upd](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_scheduled_query_rules_alert) | resource | -| [azurerm_private_dns_a_record.ingress](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_a_record) | resource | -| [helm_release.cert_mounter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.reloader](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [kubernetes_namespace.namespace](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | -| [kubernetes_namespace.namespace_system](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | -| [kubernetes_pod_disruption_budget_v1.payment_options](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/pod_disruption_budget_v1) | resource | -| [kubernetes_role_binding.deployer_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding) | resource | -| [kubernetes_role_binding.system_deployer_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding) | resource | -| [azurerm_api_management.apim](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/api_management) | data source | -| [azurerm_api_management_product.apim_api_config_product](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/api_management_product) | data source | -| [azurerm_api_management_product.apim_forwarder_product](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/api_management_product) | data source | -| [azurerm_application_insights.application_insights_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/application_insights) | data source | -| [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | -| [azurerm_eventhub_authorization_rule.pagopa_weu_core_evh_ns04_nodo_dei_pagamenti_cache_sync_reader](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | -| [azurerm_eventhub_authorization_rule.pagopa_weu_core_evh_ns04_nodo_dei_pagamenti_verify_ko_writer](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | -| [azurerm_eventhub_authorization_rule.payment_options_re_authorization_rule_writer](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | -| [azurerm_key_vault.kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault) | data source | -| [azurerm_kubernetes_cluster.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_cluster) | data source | -| [azurerm_log_analytics_workspace.log_analytics_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/log_analytics_workspace) | data source | -| [azurerm_monitor_action_group.email](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | -| [azurerm_monitor_action_group.opsgenie](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | -| [azurerm_monitor_action_group.slack](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | -| [azurerm_private_dns_zone.internal](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source | -| [azurerm_resource_group.monitor_italy_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | -| [azurerm_subnet.apim_vnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | -| [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | -| [azurerm_virtual_network.vnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/virtual_network) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [apim\_dns\_zone\_prefix](#input\_apim\_dns\_zone\_prefix) | The dns subdomain for apim. | `string` | `null` | no | -| [dns\_zone\_internal\_prefix](#input\_dns\_zone\_internal\_prefix) | The dns subdomain. | `string` | `null` | no | -| [dns\_zone\_prefix](#input\_dns\_zone\_prefix) | The wallet dns subdomain. | `string` | `null` | no | -| [domain](#input\_domain) | n/a | `string` | n/a | yes | -| [env](#input\_env) | n/a | `string` | n/a | yes | -| [env\_short](#input\_env\_short) | n/a | `string` | n/a | yes | -| [external\_domain](#input\_external\_domain) | Domain for delegation | `string` | `null` | no | -| [ingress\_load\_balancer\_ip](#input\_ingress\_load\_balancer\_ip) | ## Aks | `string` | n/a | yes | -| [instance](#input\_instance) | One of beta, prod01, prod02 | `string` | n/a | yes | -| [is\_feature\_enabled](#input\_is\_feature\_enabled) | n/a | 
object({
    paymentoptions      = bool
    paymentoptions_mock = bool
  })
| 
{
  "paymentoptions": false,
  "paymentoptions_mock": false
}
| no | -| [k8s\_kube\_config\_path\_prefix](#input\_k8s\_kube\_config\_path\_prefix) | n/a | `string` | `"~/.kube"` | no | -| [location](#input\_location) | One of westeurope, northeurope | `string` | n/a | yes | -| [location\_short](#input\_location\_short) | One of wue, neu | `string` | n/a | yes | -| [location\_string](#input\_location\_string) | One of West Europe, North Europe | `string` | n/a | yes | -| [log\_analytics\_italy\_workspace\_name](#input\_log\_analytics\_italy\_workspace\_name) | Specifies the name of the Log Analytics Workspace Italy. | `string` | n/a | yes | -| [log\_analytics\_italy\_workspace\_resource\_group\_name](#input\_log\_analytics\_italy\_workspace\_resource\_group\_name) | The name of the resource group in which the Log Analytics workspace Italy is located in. | `string` | n/a | yes | -| [log\_analytics\_workspace\_name](#input\_log\_analytics\_workspace\_name) | Specifies the name of the Log Analytics Workspace. | `string` | n/a | yes | -| [log\_analytics\_workspace\_resource\_group\_name](#input\_log\_analytics\_workspace\_resource\_group\_name) | The name of the resource group in which the Log Analytics workspace is located in. | `string` | n/a | yes | -| [monitor\_italy\_resource\_group\_name](#input\_monitor\_italy\_resource\_group\_name) | Monitor Italy resource group name | `string` | n/a | yes | -| [monitor\_resource\_group\_name](#input\_monitor\_resource\_group\_name) | Monitor resource group name | `string` | n/a | yes | -| [pod\_disruption\_budgets](#input\_pod\_disruption\_budgets) | Pod disruption budget for domain namespace | 
map(object({
    name         = optional(string, null)
    minAvailable = optional(number, null)
    matchLabels  = optional(map(any), {})
  }))
| `{}` | no | -| [prefix](#input\_prefix) | n/a | `string` | n/a | yes | -| [tags](#input\_tags) | n/a | `map(any)` | 
{
  "CreatedBy": "Terraform"
}
| no | - -## Outputs - -No outputs. - diff --git a/src/domains/gpdingestion-app/env/itn-dev/backend.ini b/src/domains/gpdingestion-app/env/itn-dev/backend.ini deleted file mode 100644 index f3ea2d530c..0000000000 --- a/src/domains/gpdingestion-app/env/itn-dev/backend.ini +++ /dev/null @@ -1 +0,0 @@ -subscription=DEV-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-app/env/itn-dev/backend.tfvars b/src/domains/gpdingestion-app/env/itn-dev/backend.tfvars deleted file mode 100644 index 127a949568..0000000000 --- a/src/domains/gpdingestion-app/env/itn-dev/backend.tfvars +++ /dev/null @@ -1,4 +0,0 @@ -resource_group_name = "terraform-state-rg" -storage_account_name = "tfinfdevpagopa" -container_name = "terraform-state" -key = "gpdingestion-app-dev.terraform.tfstate" diff --git a/src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars b/src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars deleted file mode 100644 index af815d1ab0..0000000000 --- a/src/domains/gpdingestion-app/env/itn-dev/terraform.tfvars +++ /dev/null @@ -1,55 +0,0 @@ -prefix = "pagopa" -env_short = "d" -env = "dev" -domain = "gpdingestion" -location = "italynorth" -location_short = "itn" -location_string = "Italy North" -instance = "dev" - -tags = { - CreatedBy = "Terraform" - Environment = "Dev" - Owner = "pagoPA" - Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-app" - CostCenter = "TS310 - PAGAMENTI & SERVIZI" -} - -### External resources - -monitor_italy_resource_group_name = "pagopa-d-itn-core-monitor-rg" -log_analytics_italy_workspace_name = "pagopa-d-itn-core-law" -log_analytics_italy_workspace_resource_group_name = "pagopa-d-itn-core-monitor-rg" - -monitor_resource_group_name = "pagopa-d-monitor-rg" -log_analytics_workspace_name = "pagopa-d-law" -log_analytics_workspace_resource_group_name = "pagopa-d-monitor-rg" - -external_domain = "pagopa.it" -dns_zone_internal_prefix = "internal.dev.platform" -dns_zone_prefix = "gpdingestion.itn" -apim_dns_zone_prefix = "dev.platform" -### Aks - -ingress_load_balancer_ip = "10.3.2.250" - -is_feature_enabled = { - gpdingestion = true -} - -zookeeper_replicas = 1 -zookeeper_request_memory = "512mi" -zookeeper_request_cpu = "0.5" -zookeeper_limits_memory = "512mi" -zookeeper_limits_cpu = "0.5" -zookeeper_jvm_xms = "512mi" -zookeeper_jvm_xmx = "512mi" -zookeeper_storage_size = "100Gi" -replicas = 1 -request_cpu = "0.5" -request_memory = "512mi" -limits_memory = "512mi" -limits_cpu = "0.5" -postgres_db_name = "apd" -tasks_max = "1" -container_registry = "TBD" diff --git a/src/domains/gpdingestion-app/env/itn-prod/backend.ini b/src/domains/gpdingestion-app/env/itn-prod/backend.ini deleted file mode 100644 index ddda4bb50f..0000000000 --- a/src/domains/gpdingestion-app/env/itn-prod/backend.ini +++ /dev/null @@ -1 +0,0 @@ -subscription=prod-pagoPA diff --git a/src/domains/gpdingestion-app/env/itn-prod/backend.tfvars b/src/domains/gpdingestion-app/env/itn-prod/backend.tfvars deleted file mode 100644 index 41b804626c..0000000000 --- a/src/domains/gpdingestion-app/env/itn-prod/backend.tfvars +++ /dev/null @@ -1,4 +0,0 @@ -resource_group_name = "terraform-state-rg" -storage_account_name = "tfinfprodpagopa" -container_name = "terraform-state" -key = "gpdingestion-app-prod.terraform.tfstate" diff --git a/src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars b/src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars deleted file mode 100644 index 12fe7aeef0..0000000000 --- a/src/domains/gpdingestion-app/env/itn-prod/terraform.tfvars +++ /dev/null @@ -1,64 +0,0 @@ -prefix = "pagopa" -env_short = "p" -env = "prod" -domain = "gpdingestion" -location = "italynorth" -location_short = "itn" -location_string = "Italy North" -instance = "prod" - -tags = { - CreatedBy = "Terraform" - Environment = "prod" - Owner = "pagoPA" - Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-app" - CostCenter = "TS310 - PAGAMENTI & SERVIZI" -} - -### External resources - -monitor_italy_resource_group_name = "pagopa-p-itn-core-monitor-rg" -log_analytics_italy_workspace_name = "pagopa-p-itn-core-law" -log_analytics_italy_workspace_resource_group_name = "pagopa-p-itn-core-monitor-rg" - -monitor_resource_group_name = "pagopa-p-monitor-rg" -log_analytics_workspace_name = "pagopa-p-law" -log_analytics_workspace_resource_group_name = "pagopa-p-monitor-rg" - -external_domain = "pagopa.it" -dns_zone_internal_prefix = "internal.platform" -dns_zone_prefix = "gpdingestion.itn" -apim_dns_zone_prefix = "platform" -### Aks - -ingress_load_balancer_ip = "10.3.2.250" - -is_feature_enabled = { - gpdingestion = true -} - -pod_disruption_budgets = { - "gpd-ingestion-manager" = { - minAvailable = 2 - matchLabels = { - "app.kubernetes.io/instance" = "gpd-ingestion-manager" - } - }, -} - -zookeeper_replicas = 3 -zookeeper_request_memory = "1024mi" -zookeeper_request_cpu = "1" -zookeeper_limits_memory = "1024mi" -zookeeper_limits_cpu = "1" -zookeeper_jvm_xms = "1024mi" -zookeeper_jvm_xmx = "1024mi" -zookeeper_storage_size = "100Gi" -replicas = 3 -request_cpu = "1" -request_memory = "512mi" -limits_memory = "1024mi" -limits_cpu = "1" -postgres_db_name = "apd" -tasks_max = "5" -container_registry = "TBD" diff --git a/src/domains/gpdingestion-app/env/itn-uat/backend.ini b/src/domains/gpdingestion-app/env/itn-uat/backend.ini deleted file mode 100644 index 1759a0ca0d..0000000000 --- a/src/domains/gpdingestion-app/env/itn-uat/backend.ini +++ /dev/null @@ -1 +0,0 @@ -subscription=UAT-pagoPA \ No newline at end of file diff --git a/src/domains/gpdingestion-app/env/itn-uat/backend.tfvars b/src/domains/gpdingestion-app/env/itn-uat/backend.tfvars deleted file mode 100644 index b7b0918896..0000000000 --- a/src/domains/gpdingestion-app/env/itn-uat/backend.tfvars +++ /dev/null @@ -1,4 +0,0 @@ -resource_group_name = "terraform-state-rg" -storage_account_name = "tfinfuatpagopa" -container_name = "terraform-state" -key = "gpdingestion-app-uat.terraform.tfstate" diff --git a/src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars b/src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars deleted file mode 100644 index 6ceb56d371..0000000000 --- a/src/domains/gpdingestion-app/env/itn-uat/terraform.tfvars +++ /dev/null @@ -1,55 +0,0 @@ -prefix = "pagopa" -env_short = "u" -env = "uat" -domain = "gpdingestion" -location = "italynorth" -location_short = "itn" -location_string = "Italy North" -instance = "uat" - -tags = { - CreatedBy = "Terraform" - Environment = "Uat" - Owner = "pagoPA" - Source = "https://github.com/pagopa/pagopa-infra/tree/main/src/domains/gpdingestion-app" - CostCenter = "TS310 - PAGAMENTI & SERVIZI" -} - -### External resources - -monitor_italy_resource_group_name = "pagopa-u-itn-core-monitor-rg" -log_analytics_italy_workspace_name = "pagopa-u-itn-core-law" -log_analytics_italy_workspace_resource_group_name = "pagopa-u-itn-core-monitor-rg" - -monitor_resource_group_name = "pagopa-u-monitor-rg" -log_analytics_workspace_name = "pagopa-u-law" -log_analytics_workspace_resource_group_name = "pagopa-u-monitor-rg" - -external_domain = "pagopa.it" -dns_zone_internal_prefix = "internal.uat.platform" -dns_zone_prefix = "gpdingestion.itn" -apim_dns_zone_prefix = "uat.platform" -### Aks - -ingress_load_balancer_ip = "10.3.2.250" - -is_feature_enabled = { - gpdingestion = true -} - -zookeeper_replicas = 3 -zookeeper_request_memory = "1024mi" -zookeeper_request_cpu = "1" -zookeeper_limits_memory = "1024mi" -zookeeper_limits_cpu = "1" -zookeeper_jvm_xms = "1024mi" -zookeeper_jvm_xmx = "1024mi" -zookeeper_storage_size = "100Gi" -replicas = 3 -request_cpu = "1" -request_memory = "512mi" -limits_memory = "1024mi" -limits_cpu = "1" -postgres_db_name = "apd" -tasks_max = "5" -container_registry = "TBD" diff --git a/src/domains/gpdingestion-app/helm/cert-mounter.yaml.tpl b/src/domains/gpdingestion-app/helm/cert-mounter.yaml.tpl deleted file mode 100644 index 73ee05d737..0000000000 --- a/src/domains/gpdingestion-app/helm/cert-mounter.yaml.tpl +++ /dev/null @@ -1,13 +0,0 @@ -namespace: ${NAMESPACE} -nameOverride: "" -fullnameOverride: "" - -deployment: - create: true - -kvCertificatesName: - - ${CERTIFICATE_NAME} - -keyvault: - name: "${KV_NAME}" - tenantId: "7788edaf-0346-4068-9d79-c868aed15b3d" diff --git a/src/domains/gpdingestion-app/terraform.sh b/src/domains/gpdingestion-app/terraform.sh deleted file mode 100755 index 047a7512d0..0000000000 --- a/src/domains/gpdingestion-app/terraform.sh +++ /dev/null @@ -1,324 +0,0 @@ -#!/bin/bash -############################################################ -# Terraform script for managing infrastructure on Azure -# Fingerprint: d2hhdHlvdXdhbnQ/Cg== -############################################################ -# Global variables -# Version format x.y accepted -vers="1.11" -script_name=$(basename "$0") -git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/azure/${script_name}" -tmp_file="${script_name}.new" -# Check if the third parameter exists and is a file -if [ -n "$3" ] && [ -f "$3" ]; then - FILE_ACTION=true -else - FILE_ACTION=false -fi - -# Define functions -function clean_environment() { - rm -rf .terraform - rm tfplan 2>/dev/null - echo "cleaned!" -} - -function download_tool() { - #default value - cpu_type="intel" - os_type=$(uname) - - # only on MacOS - if [ "$os_type" == "Darwin" ]; then - cpu_brand=$(sysctl -n machdep.cpu.brand_string) - if grep -q -i "intel" <<< "$cpu_brand"; then - cpu_type="intel" - else - cpu_type="arm" - fi - fi - - echo $cpu_type - tool=$1 - git_repo="https://raw.githubusercontent.com/pagopa/eng-common-scripts/main/golang/${tool}_${cpu_type}" - if ! command -v $tool &> /dev/null; then - if ! curl -sL "$git_repo" -o "$tool"; then - echo "Error downloading ${tool}" - return 1 - else - chmod +x $tool - echo "${tool} downloaded! Please note this tool WON'T be copied in your **/bin folder for safety reasons. -You need to do it yourself!" - read -p "Press enter to continue" - - - fi - fi -} - -function extract_resources() { - TF_FILE=$1 - ENV=$2 - TARGETS="" - - # Check if the file exists - if [ ! -f "$TF_FILE" ]; then - echo "File $TF_FILE does not exist." - exit 1 - fi - - # Check if the directory exists - if [ ! -d "./env/$ENV" ]; then - echo "Directory ./env/$ENV does not exist." - exit 1 - fi - - TMP_FILE=$(mktemp) - grep -E '^resource|^module' $TF_FILE > $TMP_FILE - - while read -r line ; do - TYPE=$(echo $line | cut -d '"' -f 1 | tr -d ' ') - if [ "$TYPE" == "module" ]; then - NAME=$(echo $line | cut -d '"' -f 2) - TARGETS+=" -target=\"$TYPE.$NAME\"" - else - NAME1=$(echo $line | cut -d '"' -f 2) - NAME2=$(echo $line | cut -d '"' -f 4) - TARGETS+=" -target=\"$NAME1.$NAME2\"" - fi - done < $TMP_FILE - - rm $TMP_FILE - - echo "./terraform.sh $action $ENV $TARGETS" -} - -function help_usage() { - echo "terraform.sh Version ${vers}" - echo - echo "Usage: ./script.sh [ACTION] [ENV] [OTHER OPTIONS]" - echo "es. ACTION: init, apply, plan, etc." - echo "es. ENV: dev, uat, prod, etc." - echo - echo "Available actions:" - echo " clean Remove .terraform* folders and tfplan files" - echo " help This help" - echo " list List every environment available" - echo " update Update this script if possible" - echo " summ Generate summary of Terraform plan" - echo " tflist Generate an improved output of terraform state list" - echo " tlock Generate or update the dependency lock file" - echo " * any terraform option" -} - -function init_terraform() { - if [ -n "$env" ]; then - terraform init -reconfigure -backend-config="./env/$env/backend.tfvars" - else - echo "ERROR: no env configured!" - exit 1 - fi -} - -function list_env() { - # Check if env directory exists - if [ ! -d "./env" ]; then - echo "No environment directory found" - exit 1 - fi - - # List subdirectories under env directory - env_list=$(ls -d ./env/*/ 2>/dev/null) - - # Check if there are any subdirectories - if [ -z "$env_list" ]; then - echo "No environments found" - exit 1 - fi - - # Print the list of environments - echo "Available environments:" - for env in $env_list; do - env_name=$(echo "$env" | sed 's#./env/##;s#/##') - echo "- $env_name" - done -} - -function other_actions() { - if [ -n "$env" ] && [ -n "$action" ]; then - terraform "$action" -var-file="./env/$env/terraform.tfvars" -compact-warnings $other - else - echo "ERROR: no env or action configured!" - exit 1 - fi -} - -function state_output_taint_actions() { - if [ "$action" == "tflist" ]; then - # If 'tflist' is not installed globally and there is no 'tflist' file in the current directory, - # attempt to download the 'tflist' tool - if ! command -v tflist &> /dev/null && [ ! -f "tflist" ]; then - download_tool "tflist" - if [ $? -ne 0 ]; then - echo "Error: Failed to download tflist!!" - exit 1 - else - echo "tflist downloaded!" - fi - fi - if command -v tflist &> /dev/null; then - terraform state list | tflist - else - terraform state list | ./tflist - fi - else - terraform $action $other - fi -} - - -function parse_tfplan_option() { - # Create an array to contain arguments that do not start with '-tfplan=' - local other_args=() - - # Loop over all arguments - for arg in "$@"; do - # If the argument starts with '-tfplan=', extract the file name - if [[ "$arg" =~ ^-tfplan= ]]; then - echo "${arg#*=}" - else - # If the argument does not start with '-tfplan=', add it to the other_args array - other_args+=("$arg") - fi - done - - # Print all arguments in other_args separated by spaces - echo "${other_args[@]}" -} - -function tfsummary() { - local plan_file - plan_file=$(parse_tfplan_option "$@") - if [ -z "$plan_file" ]; then - plan_file="tfplan" - fi - action="plan" - other="-out=${plan_file}" - other_actions - if [ -n "$(command -v tf-summarize)" ]; then - tf-summarize -tree "${plan_file}" - else - echo "tf-summarize is not installed" - fi - if [ "$plan_file" == "tfplan" ]; then - rm $plan_file - fi -} - -function update_script() { - # Check if the repository was cloned successfully - if ! curl -sL "$git_repo" -o "$tmp_file"; then - echo "Error cloning the repository" - rm "$tmp_file" 2>/dev/null - return 1 - fi - - # Check if a newer version exists - remote_vers=$(sed -n '8s/vers="\(.*\)"/\1/p' "$tmp_file") - if [ "$(printf '%s\n' "$vers" "$remote_vers" | sort -V | tail -n 1)" == "$vers" ]; then - echo "The local script version is equal to or newer than the remote version." - rm "$tmp_file" 2>/dev/null - return 0 - fi - - # Check the fingerprint - local_fingerprint=$(sed -n '4p' "$0") - remote_fingerprint=$(sed -n '4p' "$tmp_file") - - if [ "$local_fingerprint" != "$remote_fingerprint" ]; then - echo "The local and remote file fingerprints do not match." - rm "$tmp_file" 2>/dev/null - return 0 - fi - - # Show the current and available versions to the user - echo "Current script version: $vers" - echo "Available script version: $remote_vers" - - # Ask the user if they want to update the script - read -rp "Do you want to update the script to version $remote_vers? (y/n): " answer - - if [ "$answer" == "y" ] || [ "$answer" == "Y" ]; then - # Replace the local script with the updated version - cp "$tmp_file" "$script_name" - chmod +x "$script_name" - rm "$tmp_file" 2>/dev/null - - echo "Script successfully updated to version $remote_vers" - else - echo "Update canceled by the user" - fi - - rm "$tmp_file" 2>/dev/null -} - -# Check arguments number -if [ "$#" -lt 1 ]; then - help_usage - exit 0 -fi - -# Parse arguments -action=$1 -env=$2 -filetf=$3 -shift 2 -other=$@ - -if [ -n "$env" ]; then - # shellcheck source=/dev/null - source "./env/$env/backend.ini" - if [ -z "$(command -v az)" ]; then - echo "az not found, cannot proceed" - exit 1 - fi - az account set -s "${subscription}" -fi - -# Call appropriate function based on action -case $action in - clean) - clean_environment - ;; - ?|help|-h) - help_usage - ;; - init) - init_terraform "$other" - ;; - list) - list_env - ;; - output|state|taint|tflist) - init_terraform - state_output_taint_actions $other - ;; - summ) - init_terraform - tfsummary "$other" - ;; - tlock) - terraform providers lock -platform=windows_amd64 -platform=darwin_amd64 -platform=darwin_arm64 -platform=linux_amd64 - ;; - update) - update_script - ;; - *) - if [ "$FILE_ACTION" = true ]; then - extract_resources "$filetf" "$env" - else - init_terraform - other_actions "$other" - fi - ;; -esac diff --git a/src/domains/gpdingestion-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf similarity index 66% rename from src/domains/gpdingestion-app/05_debezium_connect.tf rename to src/domains/gps-app/05_debezium_connect.tf index 6d8bf13ccd..84fc183aa2 100644 --- a/src/domains/gpdingestion-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -1,40 +1,46 @@ -data "azurerm_key_vault_secret" "pgres_admin_login" { - name = "db-apd-user-name" +data "azurerm_key_vault_secret" "pgres_gpd_cdc_login" { + name = "cdc-logical-replication-apd-user" key_vault_id = "pagopa-${var.env_short}-gps-kv" } -data "azurerm_key_vault_secret" "pgres_admin_pwd" { - name = "db-apd-user-password" +data "azurerm_key_vault_secret" "pgres_gpd_cdc_pwd" { + name = "cdc-logical-replication-apd-pwd" key_vault_id = "pagopa-${var.env_short}-gps-kv" } +data "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" { + name = "cdc-connection-string" + namespace_name = "pagopa-${var.env_short}-itn-observ-gpd-evh" + resource_group_name = "pagopa-${var.env_short}-itn-observ-evh-rg" +} + resource "helm_release" "strimzi-kafka-operator" { name = "strimzi-kafka-operator" repository = "https://strimzi.io/charts/strimzi-kafka-operator" chart = "strimzi-kafka-operator" version = "0.8.2" - namespace = kubernetes_namespace.namespace.metadata[0].name + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name } locals { debezium_role_yaml = templatefile("${path.module}/yaml/debezium-role.yaml", { - namespace = kubernetes_namespace.namespace.metadata[0].name + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name }) debezium_rbac_yaml = templatefile("${path.module}/yaml/debezium-rbac.yaml", { - namespace = kubernetes_namespace.namespace.metadata[0].name + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name }) debezium_secrets_yaml = templatefile("${path.module}/yaml/debezium-secretes.yaml", { - namespace = kubernetes_namespace.namespace.metadata[0].name + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name username = data.azurerm_key_vault_secret.pgres_admin_login.value password = data.azurerm_key_vault_secret.pgres_admin_pwd.value }) zookeeper_yaml = templatefile("${path.module}/yaml/zookeper.yaml", { - namespace = kubernetes_namespace.namespace.metadata[0].name + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name zookeeper_replicas = var.zookeeper_replicas zookeeper_request_memory = var.zookeeper_request_memory zookeeper_request_cpu = var.zookeeper_request_cpu @@ -46,19 +52,19 @@ locals { }) kafka_connect_yaml = templatefile("${path.module}/yaml/kafka-connect.yaml", { - namespace = kubernetes_namespace.namespace.metadata[0].name + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name replicas = var.replicas request_memory = var.request_memory request_cpu = var.request_cpu limits_memory = var.limits_memory limits_cpu = var.limits_cpu - bootstrap_servers = "pagopa-${var.env_short}-${var.location_short}-${local.project}-evh.servicebus.windows.net:9092" + bootstrap_servers = "pagopa-${var.env_short}-itn-observ-gpd-evh.servicebus.windows.net:9093" eh_connection_string = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string}\";" container_registry = var.container_registry }) postgres_connector_yaml = templatefile("${path.module}/yaml/postgres-connector.yaml", { - namespace = kubernetes_namespace.namespace.metadata[0].name + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name postgres_hostname = "pagopa-${var.env_short}-gpd-postgresql.postgres.database.azure.com" postgres_port = 6432 postgres_db_name = var.postgres_db_name @@ -97,7 +103,7 @@ resource "null_resource" "wait_zookeeper" { kubectl_manifest.zookeper_manifest ] provisioner "local-exec" { - command = "while [ true ]; do STATUS=`kubectl -n ${kubernetes_namespace.namespace.metadata[0].name} get Kafka -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Zookeper SUCCEEDED\" ; break ; else echo \"Zookeeper INPROGRESS\"; sleep 3; fi ; done" + command = "while [ true ]; do STATUS=`kubectl -n gps get Kafka -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Zookeper SUCCEEDED\" ; break ; else echo \"Zookeeper INPROGRESS\"; sleep 3; fi ; done" interpreter = ["/bin/bash", "-c"] } } @@ -115,7 +121,7 @@ resource "null_resource" "wait_kafka_connect" { kubectl_manifest.kafka_connect ] provisioner "local-exec" { - command = "while [ true ]; do STATUS=`kubectl -n ${kubernetes_namespace.namespace.metadata[0].name} get KafkaConnect -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Kafka Connect SUCCEEDED\" ; break ; else echo \"Kafka Connect INPROGRESS\"; sleep 3; fi ; done" + command = "while [ true ]; do STATUS=`kubectl -n gps get KafkaConnect -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Kafka Connect SUCCEEDED\" ; break ; else echo \"Kafka Connect INPROGRESS\"; sleep 3; fi ; done" interpreter = ["/bin/bash", "-c"] } } @@ -133,7 +139,7 @@ resource "null_resource" "wait_postgres_connector" { kubectl_manifest.kafka_connect ] provisioner "local-exec" { - command = "while [ true ]; do STATUS=`kubectl -n ${kubernetes_namespace.namespace.metadata[0].name} get KafkaConnector -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Postgres Connector SUCCEEDED\" ; break ; else echo \"Postgres Connector INPROGRESS\"; sleep 3; fi ; done" + command = "while [ true ]; do STATUS=`kubectl -n gps get KafkaConnector -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Postgres Connector SUCCEEDED\" ; break ; else echo \"Postgres Connector INPROGRESS\"; sleep 3; fi ; done" interpreter = ["/bin/bash", "-c"] } } diff --git a/src/domains/gps-app/99_variables.tf b/src/domains/gps-app/99_variables.tf index 9f68170759..2b653978de 100644 --- a/src/domains/gps-app/99_variables.tf +++ b/src/domains/gps-app/99_variables.tf @@ -442,3 +442,100 @@ variable "flag_responsetime_alert" { description = "Flag to enable if payments-pull response time alert is available" default = 0 } + +### debezium kafka conn + +variable "zookeeper_replicas" { + type = number + description = "Zookeeper Replicas" + default = 1 +} + +variable "zookeeper_request_memory" { + type = string + description = "Zookeeper Request Memory" + default = "512m" +} + +variable "zookeeper_request_cpu" { + type = string + description = "Zookeeper Request CPU" + default = "0.5" +} + +variable "zookeeper_limits_memory" { + type = string + description = "Zookeeper Limit Memory" + default = "512mi" +} + +variable "zookeeper_limits_cpu" { + type = string + description = "Zookeeper Limit CPU" + default = "0.5" +} + +variable "zookeeper_jvm_xms" { + type = string + description = "Zookeeper Jvm Xms" + default = "512mi" +} + +variable "zookeeper_jvm_xmx" { + type = string + description = "Zookeeper Jvm Xmx" + default = "512mi" +} + +variable "zookeeper_storage_size" { + type = string + description = "Zookeeper Storage Size" + default = "100Gi" +} + +variable "container_registry" { + type = string + description = "Container Registry" +} + +variable "postgres_db_name" { + type = string + description = "Postgres Database Name" + default = "apd" +} + +variable "tasks_max" { + type = string + description = "Number of tasks" + default = "1" +} + +variable "replicas" { + type = number + description = "Number of replicas in cluster" + default = 1 +} + +variable "request_memory" { + type = string + description = "Connect Request Memory" + default = "512m" +} + +variable "request_cpu" { + type = string + description = "Connect Request CPU" + default = "0.5" +} + +variable "limits_memory" { + type = string + description = "Connect Limit Memory" + default = "512mi" +} + +variable "limits_cpu" { + type = string + description = "Connect Limit CPU" + default = "0.5" +} diff --git a/src/domains/gps-app/env/weu-dev/terraform.tfvars b/src/domains/gps-app/env/weu-dev/terraform.tfvars index 4a3d2f84b5..9bf6bfbb35 100644 --- a/src/domains/gps-app/env/weu-dev/terraform.tfvars +++ b/src/domains/gps-app/env/weu-dev/terraform.tfvars @@ -64,3 +64,21 @@ pgbouncer_enabled = false # WISP-dismantling-cfg create_wisp_converter = true + +### debezium kafka conn +zookeeper_replicas = 1 +zookeeper_request_memory = "512mi" +zookeeper_request_cpu = "0.5" +zookeeper_limits_memory = "512mi" +zookeeper_limits_cpu = "0.5" +zookeeper_jvm_xms = "512mi" +zookeeper_jvm_xmx = "512mi" +zookeeper_storage_size = "100Gi" +replicas = 1 +request_cpu = "0.5" +request_memory = "512mi" +limits_memory = "512mi" +limits_cpu = "0.5" +postgres_db_name = "apd" +tasks_max = "1" +container_registry = "pagopadcommonacr.azurecr.io" \ No newline at end of file diff --git a/src/domains/gpdingestion-app/set_registry_secrets.sh b/src/domains/gps-app/set_registry_secrets.sh similarity index 66% rename from src/domains/gpdingestion-app/set_registry_secrets.sh rename to src/domains/gps-app/set_registry_secrets.sh index 74212c7f55..6a89562f80 100644 --- a/src/domains/gpdingestion-app/set_registry_secrets.sh +++ b/src/domains/gps-app/set_registry_secrets.sh @@ -7,17 +7,17 @@ if [ "$#" -ne 4 ]; then fi # Assign parameters to variables -DOCKER_SERVER=$1 -DOCKER_USERNAME=$2 -DOCKER_PASSWORD=$3 +DOCKER_SERVER=$1 # acr url pagopacommonacr.azurecr.io +DOCKER_USERNAME=$2 # acr usr pagopacommonacr +DOCKER_PASSWORD=$3 # acr pwd DOCKER_EMAIL=$4 # Create the Docker registry secret using kubectl -kubectl create secret docker-registry registry-credential \ +kubectl create secret docker-registry acr-credential \ --docker-server="$DOCKER_SERVER" \ --docker-username="$DOCKER_USERNAME" \ - --docker-password="$DOCKER_PASSWORD" \ - --docker-email="$DOCKER_EMAIL" + --docker-password="$DOCKER_PASSWORD" + # --docker-email="$DOCKER_EMAIL" # Check if the secret was created successfully if [ $? -eq 0 ]; then diff --git a/src/domains/gpdingestion-app/yaml/debezium-rbac.yaml b/src/domains/gps-app/yaml/debezium-rbac.yaml similarity index 100% rename from src/domains/gpdingestion-app/yaml/debezium-rbac.yaml rename to src/domains/gps-app/yaml/debezium-rbac.yaml diff --git a/src/domains/gpdingestion-app/yaml/debezium-role.yaml b/src/domains/gps-app/yaml/debezium-role.yaml similarity index 100% rename from src/domains/gpdingestion-app/yaml/debezium-role.yaml rename to src/domains/gps-app/yaml/debezium-role.yaml diff --git a/src/domains/gpdingestion-app/yaml/debezium-secrets.yaml b/src/domains/gps-app/yaml/debezium-secrets.yaml similarity index 100% rename from src/domains/gpdingestion-app/yaml/debezium-secrets.yaml rename to src/domains/gps-app/yaml/debezium-secrets.yaml diff --git a/src/domains/gpdingestion-app/yaml/kafka-connect.yaml b/src/domains/gps-app/yaml/kafka-connect.yaml similarity index 98% rename from src/domains/gpdingestion-app/yaml/kafka-connect.yaml rename to src/domains/gps-app/yaml/kafka-connect.yaml index fdc4745ff3..761423d55f 100644 --- a/src/domains/gpdingestion-app/yaml/kafka-connect.yaml +++ b/src/domains/gps-app/yaml/kafka-connect.yaml @@ -49,6 +49,7 @@ spec: output: type: docker image: ${container_registry}/debezium-connector-postgres:latest + pushSecret: my-registry-credentials plugins: - name: debezium-postgres-connector artifacts: diff --git a/src/domains/gpdingestion-app/yaml/postgres-connector.yaml b/src/domains/gps-app/yaml/postgres-connector.yaml similarity index 68% rename from src/domains/gpdingestion-app/yaml/postgres-connector.yaml rename to src/domains/gps-app/yaml/postgres-connector.yaml index 26048026a3..985be69c36 100644 --- a/src/domains/gpdingestion-app/yaml/postgres-connector.yaml +++ b/src/domains/gps-app/yaml/postgres-connector.yaml @@ -9,17 +9,17 @@ spec: class: io.debezium.connector.postgresql.PostgresConnector tasksMax: ${tasks_max} config: - connector.class: "io.debezium.connector.postgresql.PostgresConnector", - key.converter: "org.apache.kafka.connect.json.JsonConverter", - key.converter.schemas.enable: false, - value.converter: "org.apache.kafka.connect.json.JsonConverter", - value.converter.schemas.enable: false, - tasks.max: ${tasks_max}, - database.hostname: ${postgres_hostname}, - database.port: {postgres_port}, - database.user: ${secrets:${namespace}/debezium-secret:username}, - database.password: ${secrets:${namespace}/debezium-secret:password}, - database.dbname: ${postgres_db_name}, - topic.prefix: ${postgres_topic_prefix}, - schema.include.list: "apd", + connector.class: "io.debezium.connector.postgresql.PostgresConnector" + key.converter: "org.apache.kafka.connect.json.JsonConverter" + key.converter.schemas.enable: false + value.converter: "org.apache.kafka.connect.json.JsonConverter" + value.converter.schemas.enable: false + tasks.max: ${tasks_max} + database.hostname: ${postgres_hostname} + database.port: {postgres_port} + database.user: ${secrets:${namespace}/debezium-secret:username} + database.password: ${secrets:${namespace}/debezium-secret:password} + database.dbname: ${postgres_db_name} + topic.prefix: ${postgres_topic_prefix} + schema.include.list: "apd" table.include.list: "apd.payment_option,apd.payment_option_metadata,apd.payment_position,apd.transfer,apd.transfer_metadata" diff --git a/src/domains/gpdingestion-app/yaml/zookeeper.yaml b/src/domains/gps-app/yaml/zookeeper.yaml similarity index 100% rename from src/domains/gpdingestion-app/yaml/zookeeper.yaml rename to src/domains/gps-app/yaml/zookeeper.yaml From 948d29537ea8f001d7fe26bf327b36afeaa00c5c Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 28 Oct 2024 18:50:19 +0100 Subject: [PATCH 09/55] fix --- src/domains/gps-app/.terraform.lock.hcl | 67 +++++++++---------- src/domains/gps-app/05_debezium_connect.tf | 8 +-- src/domains/gps-app/99_main.tf | 12 +++- src/domains/gps-app/set_registry_secrets.sh | 4 +- src/domains/gps-app/yaml/kafka-connect.yaml | 11 +-- .../gps-app/yaml/postgres-connector.yaml | 6 +- 6 files changed, 56 insertions(+), 52 deletions(-) diff --git a/src/domains/gps-app/.terraform.lock.hcl b/src/domains/gps-app/.terraform.lock.hcl index df945cc694..a5e96e5c05 100644 --- a/src/domains/gps-app/.terraform.lock.hcl +++ b/src/domains/gps-app/.terraform.lock.hcl @@ -6,10 +6,6 @@ provider "registry.terraform.io/azure/azapi" { constraints = "<= 1.3.0" hashes = [ "h1:OWZNYEGEIunmpxEcbGveH+kkdELQfMCUYxLt1b25UOc=", - "h1:UTQiZ34OcSms4LZ2sAa1OarDXw31OtogqBj0SwZCoDY=", - "h1:b4PzksrgRiHgOTVXIMTODOAlsvdj3uWSdCvA7lw+9ik=", - "h1:h/ZVYAapVQ+W0R4P5IK/Mvsi84jiYTggmgJHZgfVbfg=", - "h1:zpNS7i+p+MeA4h6xCbwXzcKtMeAn3je9k6J7DZQqReY=", "zh:0923b297c5b71ed584e5f3a0b2393e80244076e85102a90438159833353274b0", "zh:11fa2922aa98ca55beaf7cc33c7edbde81bbd405fdfea2955276c7f5a8537240", "zh:14af830fb6091d084bfc2711c8e9c7bf05aa3c56fe8fd8e2fb4eddeb345be88d", @@ -25,15 +21,28 @@ provider "registry.terraform.io/azure/azapi" { ] } +provider "registry.terraform.io/gavinbunney/kubectl" { + version = "1.14.0" + constraints = "1.14.0" + hashes = [ + "h1:ItrWfCZMzM2JmvDncihBMalNLutsAk7kyyxVRaipftY=", + "zh:0350f3122ff711984bbc36f6093c1fe19043173fad5a904bce27f86afe3cc858", + "zh:07ca36c7aa7533e8325b38232c77c04d6ef1081cb0bac9d56e8ccd51f12f2030", + "zh:0c351afd91d9e994a71fe64bbd1662d0024006b3493bb61d46c23ea3e42a7cf5", + "zh:39f1a0aa1d589a7e815b62b5aa11041040903b061672c4cfc7de38622866cbc4", + "zh:428d3a321043b78e23c91a8d641f2d08d6b97f74c195c654f04d2c455e017de5", + "zh:4baf5b1de2dfe9968cc0f57fd4be5a741deb5b34ee0989519267697af5f3eee5", + "zh:6131a927f9dffa014ab5ca5364ac965fe9b19830d2bbf916a5b2865b956fdfcf", + "zh:c62e0c9fd052cbf68c5c2612af4f6408c61c7e37b615dc347918d2442dd05e93", + "zh:f0beffd7ce78f49ead612e4b1aefb7cb6a461d040428f514f4f9cc4e5698ac65", + ] +} + provider "registry.terraform.io/hashicorp/azuread" { version = "2.21.0" constraints = "<= 2.21.0" hashes = [ - "h1:9gG6SWoUZZmmXbYBv6ra2RF5NYpamB9tGjsuBxrasFQ=", "h1:KbY8dRdbfTwTzEBcdOFdD50JX8CUG5Mni25D2+k1rGc=", - "h1:akcofWscEl0ecIbf7lyEqRvPfOdA5q75EZvK8uSum1c=", - "h1:p9epRqujcxIMeT9THP0oNLGe4jjMBLjT5a7RntnFDaA=", - "h1:qHYbB6LJsYPVUcd7QkZ5tU+IX+10VcUG4NzsmIuWdlE=", "zh:18c56e0478e8b3849f6d52f7e0ee495538e7fce66f22fc84a79599615e50ad1c", "zh:1b95ba8dddc46c744b2d2be7da6fafaa8ebd8368d46ff77416a95cb7d622251e", "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", @@ -53,11 +62,7 @@ provider "registry.terraform.io/hashicorp/azurerm" { version = "3.45.0" constraints = ">= 3.30.0, <= 3.45.0, <= 3.53.0" hashes = [ - "h1:4BOYXFMiLk4ozEZHUhquRnE5urebcWvaCUV3uys646o=", - "h1:V3CLlXij3vZzxw51hvCBnqriy73llPG21NjO+7sLr+U=", "h1:VQWxV5+qelZeUCjpdLvZ7iAom4RvG+fVVgK6ELvw/cs=", - "h1:WupjURkT1JPNBRzKmrSsD1Y8zhuQnL3ctKBpNLZBsLA=", - "h1:gQLNY1I5e9kcle1p/VYEWb0eteQ/t5kUfnqVu2/GBNY=", "zh:04c5dbb8845366ce5eb0dc2d55e151270cc2c0ace20993867fdae9af43b953ad", "zh:2589585da615ccae341400d45d672ee3fae413fdd88449b5befeff12a85a44b2", "zh:603869ed98fff5d9bf841a51afd9e06b628533c59356c8433aef4b15df63f5f7", @@ -78,10 +83,6 @@ provider "registry.terraform.io/hashicorp/helm" { constraints = "<= 2.5.1, <= 2.7.1" hashes = [ "h1:9yMFsXyHAo+mUuMKczNSw44HcZaf1JkMqgOUgJF1dXs=", - "h1:NasRPC0qqlpGqcF3dsSoOFu7uc5hM+zJm+okd8FgrnQ=", - "h1:a9KwjqINdNy6IsEbkHUB1vwvYfy5OJ2VxFL9/NDFLoY=", - "h1:g3CzhAURjVq69AJ1u2d3DWd+i0rSmLX+JXSIts9BV6A=", - "h1:gogHvv1qr8bPzk5y1BoeTA5dOZt47byTXWXW3CJQ5C8=", "zh:140b9748f0ad193a20d69e59d672f3c4eda8a56cede56a92f931bd3af020e2e9", "zh:17ae319466ed6538ad49e011998bb86565fe0e97bc8b9ad7c8dda46a20f90669", "zh:3a8bd723c21ba70e19f0395ed7096fc8e08bfc23366f1c3f06a9107eb37c572c", @@ -101,11 +102,7 @@ provider "registry.terraform.io/hashicorp/kubernetes" { version = "2.11.0" constraints = "<= 2.11.0" hashes = [ - "h1:T65SZhN/tQgsAsHe/G5PCgpjofi+aTKPZ+nZg6WOJpc=", - "h1:d5NamuGihJgxmtSq07kSZblnJuekkjdFdRLTDeGb0us=", "h1:lSh/Q5vX73hHL80TtGn2Vrv1UYLzlIRjC+xaCijY4ew=", - "h1:lrhK0YgF/daYiTUWAQeY3vBD7uX8gm/44ZD4GTw71r4=", - "h1:pJiAJwZKUaoAJ4x+3ONJkwEVkjrwGROCGFgj7noPO58=", "zh:143a19dd0ea3b07fc5e3d9231f3c2d01f92894385c98a67327de74c76c715843", "zh:1fc757d209e09c3cf7848e4274daa32408c07743698fbed10ee52a4a479b62b6", "zh:22dfebd0685749c51a8f765d51a1090a259778960ac1cd4f32021a325b2b9b72", @@ -122,25 +119,21 @@ provider "registry.terraform.io/hashicorp/kubernetes" { } provider "registry.terraform.io/hashicorp/null" { - version = "3.1.1" + version = "3.2.1" constraints = "<= 3.2.1" hashes = [ - "h1:1J3nqAREzuaLE7x98LEELCCaMV6BRiawHSg9MmFvfQo=", - "h1:71sNUDvmiJcijsvfXpiLCz0lXIBSsEJjMxljt7hxMhw=", - "h1:Pctug/s/2Hg5FJqjYcTM0kPyx3AoYK1MpRWO0T9V2ns=", - "h1:YvH6gTaQzGdNv+SKTZujU1O0bO+Pw6vJHOPhqgN8XNs=", - "h1:ZD4wyZ0KJzt5s2mD0xD7paJlVONNicLvZKdgtezz02I=", - "zh:063466f41f1d9fd0dd93722840c1314f046d8760b1812fa67c34de0afcba5597", - "zh:08c058e367de6debdad35fc24d97131c7cf75103baec8279aba3506a08b53faf", - "zh:73ce6dff935150d6ddc6ac4a10071e02647d10175c173cfe5dca81f3d13d8afe", + "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", + "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", + "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", + "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", + "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:8fdd792a626413502e68c195f2097352bdc6a0df694f7df350ed784741eb587e", - "zh:976bbaf268cb497400fd5b3c774d218f3933271864345f18deebe4dcbfcd6afa", - "zh:b21b78ca581f98f4cdb7a366b03ae9db23a73dfa7df12c533d7c19b68e9e72e5", - "zh:b7fc0c1615dbdb1d6fd4abb9c7dc7da286631f7ca2299fb9cd4664258ccfbff4", - "zh:d1efc942b2c44345e0c29bc976594cb7278c38cfb8897b344669eafbc3cddf46", - "zh:e356c245b3cd9d4789bab010893566acace682d7db877e52d40fc4ca34a50924", - "zh:ea98802ba92fcfa8cf12cbce2e9e7ebe999afbf8ed47fa45fc847a098d89468b", - "zh:eff8872458806499889f6927b5d954560f3d74bf20b6043409edf94d26cd906f", + "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", + "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", + "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", + "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", + "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", + "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", + "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", ] } diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index 84fc183aa2..c27cd61812 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -33,13 +33,13 @@ locals { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name }) - debezium_secrets_yaml = templatefile("${path.module}/yaml/debezium-secretes.yaml", { + debezium_secrets_yaml = templatefile("${path.module}/yaml/debezium-secrets.yaml", { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - username = data.azurerm_key_vault_secret.pgres_admin_login.value - password = data.azurerm_key_vault_secret.pgres_admin_pwd.value + username = data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value + password = data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value }) - zookeeper_yaml = templatefile("${path.module}/yaml/zookeper.yaml", { + zookeeper_yaml = templatefile("${path.module}/yaml/zookeeper.yaml", { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name zookeeper_replicas = var.zookeeper_replicas zookeeper_request_memory = var.zookeeper_request_memory diff --git a/src/domains/gps-app/99_main.tf b/src/domains/gps-app/99_main.tf index e0f14437bb..8303b763e2 100644 --- a/src/domains/gps-app/99_main.tf +++ b/src/domains/gps-app/99_main.tf @@ -24,12 +24,18 @@ terraform { source = "hashicorp/null" version = "<= 3.2.1" } + + kubectl = { + source = "gavinbunney/kubectl" + version = "1.14.0" + } } backend "azurerm" {} } provider "azurerm" { + skip_provider_registration = true features { key_vault { purge_soft_delete_on_destroy = false @@ -42,7 +48,7 @@ data "azurerm_subscription" "current" {} data "azurerm_client_config" "current" {} provider "azapi" { - + skip_provider_registration = true } provider "kubernetes" { @@ -54,3 +60,7 @@ provider "helm" { config_path = "${var.k8s_kube_config_path_prefix}/config-${local.aks_name}" } } + +module "__v3__" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" +} \ No newline at end of file diff --git a/src/domains/gps-app/set_registry_secrets.sh b/src/domains/gps-app/set_registry_secrets.sh index 6a89562f80..7da581927f 100644 --- a/src/domains/gps-app/set_registry_secrets.sh +++ b/src/domains/gps-app/set_registry_secrets.sh @@ -1,7 +1,7 @@ #!/bin/bash # Check if the required parameters are provided -if [ "$#" -ne 4 ]; then +if [ "$#" -ne 3 ]; then echo "Usage: $0 " exit 1 fi @@ -13,7 +13,7 @@ DOCKER_PASSWORD=$3 # acr pwd DOCKER_EMAIL=$4 # Create the Docker registry secret using kubectl -kubectl create secret docker-registry acr-credential \ +kubectl -n gps create secret docker-registry acr-credential \ --docker-server="$DOCKER_SERVER" \ --docker-username="$DOCKER_USERNAME" \ --docker-password="$DOCKER_PASSWORD" diff --git a/src/domains/gps-app/yaml/kafka-connect.yaml b/src/domains/gps-app/yaml/kafka-connect.yaml index 761423d55f..b22da3e6b8 100644 --- a/src/domains/gps-app/yaml/kafka-connect.yaml +++ b/src/domains/gps-app/yaml/kafka-connect.yaml @@ -11,11 +11,11 @@ spec: bootstrapServers: ${bootstrap_servers} resources: requests: - memory: $connect_request_memory} - cpu: ${connect_request_cpu} + memory: ${request_memory} + cpu: ${request_cpu} limits: - memory: ${connect_limits_memory} - cpu: ${connect_limits_cpu} + memory: ${limits_memory} + cpu: ${limits_cpu} config: config.providers: secrets config.providers.secrets.class: io.strimzi.kafka.KubernetesSecretConfigProvider @@ -49,7 +49,8 @@ spec: output: type: docker image: ${container_registry}/debezium-connector-postgres:latest - pushSecret: my-registry-credentials + # src/domains/gps-app/set_registry_secrets.sh + pushSecret: acr-credentials plugins: - name: debezium-postgres-connector artifacts: diff --git a/src/domains/gps-app/yaml/postgres-connector.yaml b/src/domains/gps-app/yaml/postgres-connector.yaml index 985be69c36..9418de5d28 100644 --- a/src/domains/gps-app/yaml/postgres-connector.yaml +++ b/src/domains/gps-app/yaml/postgres-connector.yaml @@ -16,9 +16,9 @@ spec: value.converter.schemas.enable: false tasks.max: ${tasks_max} database.hostname: ${postgres_hostname} - database.port: {postgres_port} - database.user: ${secrets:${namespace}/debezium-secret:username} - database.password: ${secrets:${namespace}/debezium-secret:password} + database.port: ${postgres_port} + database.user: ${secrets}:${namespace}/debezium-secret:username} + database.password: ${secrets}:${namespace}/debezium-secret:password} database.dbname: ${postgres_db_name} topic.prefix: ${postgres_topic_prefix} schema.include.list: "apd" From f7b85939ded8332131e6b0f920b888719f7d9054 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Tue, 29 Oct 2024 16:11:21 +0100 Subject: [PATCH 10/55] fix --- src/domains/gps-app/05_debezium_connect.tf | 84 ++++++++++--------- ...ts.yaml => debezium-secrets.yaml_NOT_USED} | 0 .../gps-app/yaml/postgres-connector.yaml | 6 +- 3 files changed, 48 insertions(+), 42 deletions(-) rename src/domains/gps-app/yaml/{debezium-secrets.yaml => debezium-secrets.yaml_NOT_USED} (100%) diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index c27cd61812..105191027b 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -1,26 +1,27 @@ +# https://debezium.io/documentation/reference/stable/operations/kubernetes.html#_creating_a_debezium_connector data "azurerm_key_vault_secret" "pgres_gpd_cdc_login" { name = "cdc-logical-replication-apd-user" - key_vault_id = "pagopa-${var.env_short}-gps-kv" + key_vault_id = data.azurerm_key_vault.kv.id } data "azurerm_key_vault_secret" "pgres_gpd_cdc_pwd" { name = "cdc-logical-replication-apd-pwd" - key_vault_id = "pagopa-${var.env_short}-gps-kv" + key_vault_id = data.azurerm_key_vault.kv.id } data "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" { - name = "cdc-connection-string" + name = "cdc-gpd-connection-string" namespace_name = "pagopa-${var.env_short}-itn-observ-gpd-evh" resource_group_name = "pagopa-${var.env_short}-itn-observ-evh-rg" } +# https://github.com/strimzi/strimzi-kafka-operator/tree/main/helm-charts/helm3/strimzi-kafka-operator resource "helm_release" "strimzi-kafka-operator" { name = "strimzi-kafka-operator" - repository = "https://strimzi.io/charts/strimzi-kafka-operator" chart = "strimzi-kafka-operator" - version = "0.8.2" - - namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + repository = "oci://quay.io/strimzi-helm" + version = "0.43.0" + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name } locals { @@ -33,43 +34,45 @@ locals { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name }) - debezium_secrets_yaml = templatefile("${path.module}/yaml/debezium-secrets.yaml", { - namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - username = data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value - password = data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value - }) + # debezium_secrets_yaml = templatefile("${path.module}/yaml/debezium-secrets.yaml", { + # namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + # username = data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value + # password = data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value + # }) zookeeper_yaml = templatefile("${path.module}/yaml/zookeeper.yaml", { - namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - zookeeper_replicas = var.zookeeper_replicas - zookeeper_request_memory = var.zookeeper_request_memory - zookeeper_request_cpu = var.zookeeper_request_cpu - zookeeper_limits_memory = var.zookeeper_limits_memory - zookeeper_limits_cpu = var.zookeeper_limits_cpu - zookeeper_jvm_xms = var.zookeeper_jvm_xms - zookeeper_jvm_xmx = var.zookeeper_jvm_xmx - zookeeper_storage_size = var.zookeeper_storage_size + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + zookeeper_replicas = var.zookeeper_replicas + zookeeper_request_memory = var.zookeeper_request_memory + zookeeper_request_cpu = var.zookeeper_request_cpu + zookeeper_limits_memory = var.zookeeper_limits_memory + zookeeper_limits_cpu = var.zookeeper_limits_cpu + zookeeper_jvm_xms = var.zookeeper_jvm_xms + zookeeper_jvm_xmx = var.zookeeper_jvm_xmx + zookeeper_storage_size = var.zookeeper_storage_size }) kafka_connect_yaml = templatefile("${path.module}/yaml/kafka-connect.yaml", { - namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - replicas = var.replicas - request_memory = var.request_memory - request_cpu = var.request_cpu - limits_memory = var.limits_memory - limits_cpu = var.limits_cpu - bootstrap_servers = "pagopa-${var.env_short}-itn-observ-gpd-evh.servicebus.windows.net:9093" - eh_connection_string = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string}\";" - container_registry = var.container_registry + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + replicas = var.replicas + request_memory = var.request_memory + request_cpu = var.request_cpu + limits_memory = var.limits_memory + limits_cpu = var.limits_cpu + bootstrap_servers = "pagopa-${var.env_short}-itn-observ-gpd-evh.servicebus.windows.net:9093" + eh_connection_string = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string}\";" + container_registry = var.container_registry }) postgres_connector_yaml = templatefile("${path.module}/yaml/postgres-connector.yaml", { - namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - postgres_hostname = "pagopa-${var.env_short}-gpd-postgresql.postgres.database.azure.com" - postgres_port = 6432 - postgres_db_name = var.postgres_db_name + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + postgres_hostname = "pagopa-${var.env_short}-gpd-postgresql.postgres.database.azure.com" + postgres_port = 6432 + postgres_db_name = var.postgres_db_name postgres_topic_prefix = "gpd" - tasks_max = var.tasks_max + postgres_username = data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value + postgres_password = data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value + tasks_max = var.tasks_max }) } @@ -79,13 +82,14 @@ resource "kubectl_manifest" "debezium_role" { yaml_body = local.debezium_role_yaml } -resource "kubectl_manifest" "debezium_secrets" { - force_conflicts = true - yaml_body = local.debezium_secrets_yaml -} +# resource "kubectl_manifest" "debezium_secrets" { +# force_conflicts = true +# yaml_body = local.debezium_secrets_yaml +# } resource "kubectl_manifest" "debezoum_rbac" { - depends_on = [kubectl_manifest.debezium_role, kubectl_manifest.debezium_secrets] + # depends_on = [kubectl_manifest.debezium_role, kubectl_manifest.debezium_secrets] + depends_on = [kubectl_manifest.debezium_role] force_conflicts = true yaml_body = local.debezium_rbac_yaml } diff --git a/src/domains/gps-app/yaml/debezium-secrets.yaml b/src/domains/gps-app/yaml/debezium-secrets.yaml_NOT_USED similarity index 100% rename from src/domains/gps-app/yaml/debezium-secrets.yaml rename to src/domains/gps-app/yaml/debezium-secrets.yaml_NOT_USED diff --git a/src/domains/gps-app/yaml/postgres-connector.yaml b/src/domains/gps-app/yaml/postgres-connector.yaml index 9418de5d28..57d4914500 100644 --- a/src/domains/gps-app/yaml/postgres-connector.yaml +++ b/src/domains/gps-app/yaml/postgres-connector.yaml @@ -17,9 +17,11 @@ spec: tasks.max: ${tasks_max} database.hostname: ${postgres_hostname} database.port: ${postgres_port} - database.user: ${secrets}:${namespace}/debezium-secret:username} - database.password: ${secrets}:${namespace}/debezium-secret:password} + database.user: ${postgres_username} + database.password: ${postgres_password} database.dbname: ${postgres_db_name} topic.prefix: ${postgres_topic_prefix} schema.include.list: "apd" table.include.list: "apd.payment_option,apd.payment_option_metadata,apd.payment_position,apd.transfer,apd.transfer_metadata" + plugin.name: "pgoutput" + publication.autocreate.mode: "disabled" # shall be create before From 1c12976b44f4a9f24e18e71417800e64b0cddcd6 Mon Sep 17 00:00:00 2001 From: acialini Date: Wed, 30 Oct 2024 16:08:35 +0100 Subject: [PATCH 11/55] [PPANTT-168] feat: updated terraform vars --- .../gps-app/env/weu-dev/terraform.tfvars | 22 +++++++++---------- .../gps-app/env/weu-prod/terraform.tfvars | 18 +++++++++++++++ .../gps-app/env/weu-uat/terraform.tfvars | 18 +++++++++++++++ 3 files changed, 47 insertions(+), 11 deletions(-) diff --git a/src/domains/gps-app/env/weu-dev/terraform.tfvars b/src/domains/gps-app/env/weu-dev/terraform.tfvars index 9bf6bfbb35..30e4ecb11b 100644 --- a/src/domains/gps-app/env/weu-dev/terraform.tfvars +++ b/src/domains/gps-app/env/weu-dev/terraform.tfvars @@ -67,18 +67,18 @@ create_wisp_converter = true ### debezium kafka conn zookeeper_replicas = 1 -zookeeper_request_memory = "512mi" -zookeeper_request_cpu = "0.5" -zookeeper_limits_memory = "512mi" -zookeeper_limits_cpu = "0.5" -zookeeper_jvm_xms = "512mi" -zookeeper_jvm_xmx = "512mi" +zookeeper_request_memory = "512Mi" +zookeeper_request_cpu = 0.5 +zookeeper_limits_memory = "512Mi" +zookeeper_limits_cpu = 0.5 +zookeeper_jvm_xms = "512m" +zookeeper_jvm_xmx = "512m" zookeeper_storage_size = "100Gi" replicas = 1 -request_cpu = "0.5" -request_memory = "512mi" -limits_memory = "512mi" -limits_cpu = "0.5" +request_cpu = 0.5 +request_memory = "512Mi" +limits_memory = "512Mi" +limits_cpu = 0.5 postgres_db_name = "apd" tasks_max = "1" -container_registry = "pagopadcommonacr.azurecr.io" \ No newline at end of file +container_registry = "pagopadcommonacr.azurecr.io" diff --git a/src/domains/gps-app/env/weu-prod/terraform.tfvars b/src/domains/gps-app/env/weu-prod/terraform.tfvars index bee8af816d..c04a5ee3fd 100644 --- a/src/domains/gps-app/env/weu-prod/terraform.tfvars +++ b/src/domains/gps-app/env/weu-prod/terraform.tfvars @@ -111,3 +111,21 @@ fn_app_storage_account_info = { access_tier = "Hot" advanced_threat_protection_enable = true } + +### debezium kafka conn +zookeeper_replicas = 3 +zookeeper_request_memory = "512Mi" +zookeeper_request_cpu = 0.5 +zookeeper_limits_memory = "1024Mi" +zookeeper_limits_cpu = 1 +zookeeper_jvm_xms = "512m" +zookeeper_jvm_xmx = "1024m" +zookeeper_storage_size = "100Gi" +replicas = 3 +request_cpu = 0.5 +request_memory = "512Mi" +limits_memory = "1024Mi" +limits_cpu = 1 +postgres_db_name = "apd" +tasks_max = "1" +container_registry = "pagopadcommonacr.azurecr.io" diff --git a/src/domains/gps-app/env/weu-uat/terraform.tfvars b/src/domains/gps-app/env/weu-uat/terraform.tfvars index d47d5bd139..611b94f378 100644 --- a/src/domains/gps-app/env/weu-uat/terraform.tfvars +++ b/src/domains/gps-app/env/weu-uat/terraform.tfvars @@ -64,3 +64,21 @@ pgbouncer_enabled = true # WISP-dismantling-cfg create_wisp_converter = true + +### debezium kafka conn +zookeeper_replicas = 3 +zookeeper_request_memory = "512Mi" +zookeeper_request_cpu = 0.5 +zookeeper_limits_memory = "1024Mi" +zookeeper_limits_cpu = 1 +zookeeper_jvm_xms = "512m" +zookeeper_jvm_xmx = "1024m" +zookeeper_storage_size = "100Gi" +replicas = 3 +request_cpu = 0.5 +request_memory = "512Mi" +limits_memory = "1024Mi" +limits_cpu = 1 +postgres_db_name = "apd" +tasks_max = "1" +container_registry = "pagopadcommonacr.azurecr.io" From 1381777a2004bc9583215490a234c20fa80cec92 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Wed, 30 Oct 2024 17:47:43 +0100 Subject: [PATCH 12/55] fix --- src/domains/gps-app/05_debezium_connect.tf | 59 ++++++++++--------- .../gps-app/env/weu-dev/terraform.tfvars | 6 +- src/domains/gps-app/yaml/kafka-connect.yaml | 18 +++--- src/domains/gps-app/yaml/zookeeper.yaml | 8 +-- 4 files changed, 47 insertions(+), 44 deletions(-) diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index 105191027b..5a5b14e6e2 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -40,17 +40,20 @@ locals { # password = data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value # }) - zookeeper_yaml = templatefile("${path.module}/yaml/zookeeper.yaml", { - namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - zookeeper_replicas = var.zookeeper_replicas - zookeeper_request_memory = var.zookeeper_request_memory - zookeeper_request_cpu = var.zookeeper_request_cpu - zookeeper_limits_memory = var.zookeeper_limits_memory - zookeeper_limits_cpu = var.zookeeper_limits_cpu - zookeeper_jvm_xms = var.zookeeper_jvm_xms - zookeeper_jvm_xmx = var.zookeeper_jvm_xmx - zookeeper_storage_size = var.zookeeper_storage_size - }) + # zookeeper_yaml = templatefile("${path.module}/yaml/zookeeper.yaml", { + # namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + # zookeeper_replicas = var.zookeeper_replicas + # zookeeper_request_memory = var.zookeeper_request_memory + # zookeeper_request_cpu = var.zookeeper_request_cpu + # zookeeper_limits_memory = var.zookeeper_limits_memory + # zookeeper_limits_cpu = var.zookeeper_limits_cpu + # zookeeper_jvm_xms = var.zookeeper_jvm_xms + # zookeeper_jvm_xmx = var.zookeeper_jvm_xmx + # zookeeper_storage_size = var.zookeeper_storage_size + # }) + + # Az config + # https://learn.microsoft.com/it-it/azure/event-hubs/event-hubs-kafka-connect-debezium#configure-kafka-connect-for-event-hubs kafka_connect_yaml = templatefile("${path.module}/yaml/kafka-connect.yaml", { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name @@ -60,7 +63,7 @@ locals { limits_memory = var.limits_memory limits_cpu = var.limits_cpu bootstrap_servers = "pagopa-${var.env_short}-itn-observ-gpd-evh.servicebus.windows.net:9093" - eh_connection_string = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string}\";" + eh_connection_string = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string}\"" container_registry = var.container_registry }) @@ -94,23 +97,23 @@ resource "kubectl_manifest" "debezoum_rbac" { yaml_body = local.debezium_rbac_yaml } -resource "kubectl_manifest" "zookeper_manifest" { - depends_on = [ - helm_release.strimzi-kafka-operator - ] - force_conflicts = true - yaml_body = local.zookeeper_yaml -} +# resource "kubectl_manifest" "zookeper_manifest" { +# depends_on = [ +# helm_release.strimzi-kafka-operator +# ] +# force_conflicts = true +# yaml_body = local.zookeeper_yaml +# } -resource "null_resource" "wait_zookeeper" { - depends_on = [ - kubectl_manifest.zookeper_manifest - ] - provisioner "local-exec" { - command = "while [ true ]; do STATUS=`kubectl -n gps get Kafka -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Zookeper SUCCEEDED\" ; break ; else echo \"Zookeeper INPROGRESS\"; sleep 3; fi ; done" - interpreter = ["/bin/bash", "-c"] - } -} +# resource "null_resource" "wait_zookeeper" { +# depends_on = [ +# kubectl_manifest.zookeper_manifest +# ] +# provisioner "local-exec" { +# command = "while [ true ]; do STATUS=`kubectl -n gps get Kafka -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Zookeper SUCCEEDED\" ; break ; else echo \"Zookeeper INPROGRESS\"; sleep 3; fi ; done" +# interpreter = ["/bin/bash", "-c"] +# } +# } resource "kubectl_manifest" "kafka_connect" { depends_on = [ diff --git a/src/domains/gps-app/env/weu-dev/terraform.tfvars b/src/domains/gps-app/env/weu-dev/terraform.tfvars index 30e4ecb11b..d629af5bd7 100644 --- a/src/domains/gps-app/env/weu-dev/terraform.tfvars +++ b/src/domains/gps-app/env/weu-dev/terraform.tfvars @@ -66,11 +66,11 @@ pgbouncer_enabled = false create_wisp_converter = true ### debezium kafka conn -zookeeper_replicas = 1 +zookeeper_replicas = "1" zookeeper_request_memory = "512Mi" -zookeeper_request_cpu = 0.5 +zookeeper_request_cpu = "0.5" zookeeper_limits_memory = "512Mi" -zookeeper_limits_cpu = 0.5 +zookeeper_limits_cpu = "0.5" zookeeper_jvm_xms = "512m" zookeeper_jvm_xmx = "512m" zookeeper_storage_size = "100Gi" diff --git a/src/domains/gps-app/yaml/kafka-connect.yaml b/src/domains/gps-app/yaml/kafka-connect.yaml index b22da3e6b8..6b9f0adfe7 100644 --- a/src/domains/gps-app/yaml/kafka-connect.yaml +++ b/src/domains/gps-app/yaml/kafka-connect.yaml @@ -6,16 +6,16 @@ metadata: annotations: strimzi.io/use-connector-resources: "true" spec: - version: 3.1.0 + version: 3.7.0 replicas: ${replicas} bootstrapServers: ${bootstrap_servers} resources: requests: - memory: ${request_memory} - cpu: ${request_cpu} + memory: "${request_memory}" + cpu: "${request_cpu}" limits: - memory: ${limits_memory} - cpu: ${limits_cpu} + memory: "${limits_memory}" + cpu: "${limits_cpu}" config: config.providers: secrets config.providers.secrets.class: io.strimzi.kafka.KubernetesSecretConfigProvider @@ -38,19 +38,19 @@ spec: internal.value.converter.schemas.enable: false security.protocol: SASL_SSL sasl.mechanism: PLAIN - sasl.jaas.config: ${eh_connection_string} + sasl.jaas.config: '${eh_connection_string}' producer.security.protocol: SASL_SSL producer.sasl.mechanism: PLAIN - producer.sasl.jaas.config: ${eh_connection_string} + producer.sasl.jaas.config: '${eh_connection_string}' consumer.security.protocol: SASL_SSL consumer.sasl.mechanism: PLAIN - consumer.sasl.jaas.config: ${eh_connection_string} + consumer.sasl.jaas.config: '${eh_connection_string}' build: output: type: docker image: ${container_registry}/debezium-connector-postgres:latest # src/domains/gps-app/set_registry_secrets.sh - pushSecret: acr-credentials + pushSecret: acr-credential plugins: - name: debezium-postgres-connector artifacts: diff --git a/src/domains/gps-app/yaml/zookeeper.yaml b/src/domains/gps-app/yaml/zookeeper.yaml index 7bbbe7b256..fc9c042e5a 100644 --- a/src/domains/gps-app/yaml/zookeeper.yaml +++ b/src/domains/gps-app/yaml/zookeeper.yaml @@ -12,11 +12,11 @@ spec: zookeeper.root.logger: INFO resources: requests: - memory: ${zookeeper_request_memory} - cpu: ${zookeeper_request_cpu} + memory: "${zookeeper_request_memory}" + cpu: "${zookeeper_request_cpu}" limits: - memory: ${zookeeper_limits_memory} - cpu: ${zookeeper_limits_cpu} + memory: "${zookeeper_limits_memory}" + cpu: "${zookeeper_limits_cpu}" jvmOptions: -Xms: ${zookeeper_jvm_xms} -Xmx: ${zookeeper_jvm_xmx} From 6594ea68d386e99f95f1342a0e97223f013baaf8 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Wed, 30 Oct 2024 18:25:08 +0100 Subject: [PATCH 13/55] fix --- src/domains/gps-app/05_debezium_connect.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index 5a5b14e6e2..011bf498b7 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -63,7 +63,7 @@ locals { limits_memory = var.limits_memory limits_cpu = var.limits_cpu bootstrap_servers = "pagopa-${var.env_short}-itn-observ-gpd-evh.servicebus.windows.net:9093" - eh_connection_string = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string}\"" + eh_connection_string = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string};\"" container_registry = var.container_registry }) From ecb91b1ae21b137b36428d099914ca43ff6edf14 Mon Sep 17 00:00:00 2001 From: acialini Date: Thu, 31 Oct 2024 09:28:38 +0100 Subject: [PATCH 14/55] [PPANTT-168] feat: updated 05_debezium_connect.tf, debezium-secrets.yaml, kafka-connect.yaml --- src/domains/gps-app/05_debezium_connect.tf | 12 ++++++------ ...rets.yaml_NOT_USED => debezium-secrets.yaml} | 1 + src/domains/gps-app/yaml/kafka-connect.yaml | 17 ++++++++--------- 3 files changed, 15 insertions(+), 15 deletions(-) rename src/domains/gps-app/yaml/{debezium-secrets.yaml_NOT_USED => debezium-secrets.yaml} (78%) diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index 011bf498b7..cd93b48b26 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -34,11 +34,12 @@ locals { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name }) - # debezium_secrets_yaml = templatefile("${path.module}/yaml/debezium-secrets.yaml", { - # namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - # username = data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value - # password = data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value - # }) + debezium_secrets_yaml = templatefile("${path.module}/yaml/debezium-secrets.yaml", { + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + username = data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value + password = data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value + connection_string = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string};\"" + }) # zookeeper_yaml = templatefile("${path.module}/yaml/zookeeper.yaml", { # namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name @@ -63,7 +64,6 @@ locals { limits_memory = var.limits_memory limits_cpu = var.limits_cpu bootstrap_servers = "pagopa-${var.env_short}-itn-observ-gpd-evh.servicebus.windows.net:9093" - eh_connection_string = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string};\"" container_registry = var.container_registry }) diff --git a/src/domains/gps-app/yaml/debezium-secrets.yaml_NOT_USED b/src/domains/gps-app/yaml/debezium-secrets.yaml similarity index 78% rename from src/domains/gps-app/yaml/debezium-secrets.yaml_NOT_USED rename to src/domains/gps-app/yaml/debezium-secrets.yaml index 4d996e3e90..eaa7bd9566 100644 --- a/src/domains/gps-app/yaml/debezium-secrets.yaml_NOT_USED +++ b/src/domains/gps-app/yaml/debezium-secrets.yaml @@ -7,3 +7,4 @@ type: Opaque data: username: ${username} password: ${password} + connection_string: ${connection_string} diff --git a/src/domains/gps-app/yaml/kafka-connect.yaml b/src/domains/gps-app/yaml/kafka-connect.yaml index 6b9f0adfe7..fec54494c8 100644 --- a/src/domains/gps-app/yaml/kafka-connect.yaml +++ b/src/domains/gps-app/yaml/kafka-connect.yaml @@ -9,6 +9,14 @@ spec: version: 3.7.0 replicas: ${replicas} bootstrapServers: ${bootstrap_servers} + tls: + trustedCertificates: [] + authentication: + type: plain + username: "$ConnectionString" + passwordSecret: + secretName: debezium-secret + password: connection_string resources: requests: memory: "${request_memory}" @@ -36,15 +44,6 @@ spec: internal.value.converter: org.apache.kafka.connect.json.JsonConverter internal.key.converter.schemas.enable: false internal.value.converter.schemas.enable: false - security.protocol: SASL_SSL - sasl.mechanism: PLAIN - sasl.jaas.config: '${eh_connection_string}' - producer.security.protocol: SASL_SSL - producer.sasl.mechanism: PLAIN - producer.sasl.jaas.config: '${eh_connection_string}' - consumer.security.protocol: SASL_SSL - consumer.sasl.mechanism: PLAIN - consumer.sasl.jaas.config: '${eh_connection_string}' build: output: type: docker From 310a5201c050f98abf2643f1cd2c29863c405c0d Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Thu, 31 Oct 2024 09:45:34 +0100 Subject: [PATCH 15/55] fix --- src/domains/gps-app/05_debezium_connect.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index cd93b48b26..9ef20d096e 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -85,10 +85,10 @@ resource "kubectl_manifest" "debezium_role" { yaml_body = local.debezium_role_yaml } -# resource "kubectl_manifest" "debezium_secrets" { -# force_conflicts = true -# yaml_body = local.debezium_secrets_yaml -# } +resource "kubectl_manifest" "debezium_secrets" { + force_conflicts = true + yaml_body = local.debezium_secrets_yaml +} resource "kubectl_manifest" "debezoum_rbac" { # depends_on = [kubectl_manifest.debezium_role, kubectl_manifest.debezium_secrets] From 6328de1a562b2bc7b1c5dd7aa6bf3d974631bf58 Mon Sep 17 00:00:00 2001 From: acialini Date: Thu, 31 Oct 2024 10:07:28 +0100 Subject: [PATCH 16/55] [PPANTT-168] feat: updated 05_debezium_connect.tf, debezium-secrets.yaml, kafka-connect.yaml --- src/domains/gps-app/05_debezium_connect.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index cd93b48b26..c0724bb3d9 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -36,9 +36,9 @@ locals { debezium_secrets_yaml = templatefile("${path.module}/yaml/debezium-secrets.yaml", { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - username = data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value - password = data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value - connection_string = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$ConnectionString\" password=\"${data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string};\"" + username = base64encode(data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value) + password = base64encode(data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value) + connection_string = base64encode(data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string) }) # zookeeper_yaml = templatefile("${path.module}/yaml/zookeeper.yaml", { From 5623b13a1b595eb5366b6112d646b731f5edb3a5 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Thu, 31 Oct 2024 16:10:20 +0100 Subject: [PATCH 17/55] fix --- src/domains/gps-app/05_debezium_connect.tf | 7 +- .../observability/03_eventhub_msg_gdp.tf | 14 ++ src/domains/observability/99_main.tf | 4 + .../observability/env/dev/terraform.tfvars | 221 ++++++++++-------- .../observability/gpd_evh_create__az.sh | 91 ++++++++ .../observability/gpd_evh_delete__az.sh | 60 +++++ 6 files changed, 292 insertions(+), 105 deletions(-) create mode 100644 src/domains/observability/gpd_evh_create__az.sh create mode 100644 src/domains/observability/gpd_evh_delete__az.sh diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index 24edec1a8f..bba1836fa0 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -69,10 +69,11 @@ locals { postgres_connector_yaml = templatefile("${path.module}/yaml/postgres-connector.yaml", { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - postgres_hostname = "pagopa-${var.env_short}-gpd-postgresql.postgres.database.azure.com" - postgres_port = 6432 + postgres_hostname = "pagopa-${var.env_short}-gpd-pgflex.postgres.database.azure.com" + + postgres_port = 5432 postgres_db_name = var.postgres_db_name - postgres_topic_prefix = "gpd" + postgres_topic_prefix = "azcligpd" postgres_username = data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value postgres_password = data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value tasks_max = var.tasks_max diff --git a/src/domains/observability/03_eventhub_msg_gdp.tf b/src/domains/observability/03_eventhub_msg_gdp.tf index 6a4e041ae8..955cfb4161 100644 --- a/src/domains/observability/03_eventhub_msg_gdp.tf +++ b/src/domains/observability/03_eventhub_msg_gdp.tf @@ -63,3 +63,17 @@ resource "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" manage = false } +# MS doc configure-cleanup-policy https://learn.microsoft.com/en-us/azure/event-hubs/configure-event-hub-properties#configure-cleanup-policy +# ISSUE https://github.com/hashicorp/terraform-provider-azurerm/issues/22155 +# MS doc create evh via TF https://learn.microsoft.com/en-us/azure/templates/microsoft.eventhub/namespaces/eventhubs?pivots=deployment-language-terraform + +# Ex : +# az eventhubs eventhub create \ +# -g pagopa-d-itn-observ-evh-rg \ +# -n "prova" \ +# --namespace-name pagopa-d-itn-observ-gpd-evh \ +# --cleanup-policy "Compact" \ +# --status "Active" \ +# --partition-count 1 \ +# --retention-time 24 + diff --git a/src/domains/observability/99_main.tf b/src/domains/observability/99_main.tf index 940c238ee5..28cc1903cf 100644 --- a/src/domains/observability/99_main.tf +++ b/src/domains/observability/99_main.tf @@ -12,6 +12,10 @@ terraform { source = "hashicorp/null" version = "= 3.1.1" } + azapi = { + source = "azure/azapi" + version = "<= 2.0.1" + } } backend "azurerm" {} diff --git a/src/domains/observability/env/dev/terraform.tfvars b/src/domains/observability/env/dev/terraform.tfvars index 8c8e4271ab..31a750197e 100644 --- a/src/domains/observability/env/dev/terraform.tfvars +++ b/src/domains/observability/env/dev/terraform.tfvars @@ -261,119 +261,136 @@ ehns_metric_alerts = { }, } + + + eventhubs_gpd = [ - { - name = "connect-cluster-offsets" # debezium internal use + { + name = "test-evh" # test partitions = 1 message_retention = 1 - consumers = ["connect-cluster-offsets"] + consumers = ["test-evh"] keys = [ { - name = "connect-cluster-offsets" + name = "test-evh" listen = true send = true manage = false } ] }, - { - name = "connect-cluster-status" # debezium internal use - partitions = 1 - message_retention = 1 - consumers = ["connect-cluster-offsets"] - keys = [ - { - name = "connect-cluster-status" - listen = true - send = true - manage = false - } - ] - }, - { - name = "connect-cluster-configs" # debezium internal use - partitions = 1 - message_retention = 1 - consumers = ["connect-cluster-configs"] - keys = [ - { - name = "connect-cluster-configs" - listen = true - send = true - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.payment_option" - partitions = 1 - message_retention = 1 - consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] - keys = [ - { - name = "gpd-ingestion.apd.payment_option-rx-dl" - listen = true - send = false - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.payment_option_metadata" - partitions = 1 - message_retention = 1 - consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] - keys = [ - { - name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" - listen = true - send = false - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.payment_position" - partitions = 1 - message_retention = 1 - consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] - keys = [ - { - name = "gpd-ingestion.apd.payment_position-rx-dl" - listen = true - send = true - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.transfer" - partitions = 1 - message_retention = 1 - consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] - keys = [ - { - name = "gpd-ingestion.apd.transfer-rx-dl" - listen = true - send = false - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.transfer_metadata" - partitions = 1 - message_retention = 1 - consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] - keys = [ - { - name = "gpd-ingestion.apd.transfer_metadata-rx-dl" - listen = true - send = false - manage = false - } - ] - }, + # { + # name = "connect-cluster-offsets" # debezium internal use + # partitions = 1 + # message_retention = 1 + # consumers = ["connect-cluster-offsets"] + # keys = [ + # { + # name = "connect-cluster-offsets" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "connect-cluster-status" # debezium internal use + # partitions = 1 + # message_retention = 1 + # consumers = ["connect-cluster-offsets"] + # keys = [ + # { + # name = "connect-cluster-status" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "connect-cluster-configs" # debezium internal use + # partitions = 1 + # message_retention = 1 + # consumers = ["connect-cluster-configs"] + # keys = [ + # { + # name = "connect-cluster-configs" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.payment_option" + # partitions = 1 + # message_retention = 1 + # consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] + # keys = [ + # { + # name = "gpd-ingestion.apd.payment_option-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.payment_option_metadata" + # partitions = 1 + # message_retention = 1 + # consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.payment_position" + # partitions = 1 + # message_retention = 1 + # consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.payment_position-rx-dl" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.transfer" + # partitions = 1 + # message_retention = 1 + # consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.transfer-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.transfer_metadata" + # partitions = 1 + # message_retention = 1 + # consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.transfer_metadata-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, ] diff --git a/src/domains/observability/gpd_evh_create__az.sh b/src/domains/observability/gpd_evh_create__az.sh new file mode 100644 index 0000000000..4492b72f74 --- /dev/null +++ b/src/domains/observability/gpd_evh_create__az.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +set -e + +echo ">>>>>> 1" + +az eventhubs eventhub create \ +-g pagopa-d-itn-observ-evh-rg \ +-n "connect-cluster-offsets" \ +--namespace-name pagopa-d-itn-observ-gpd-evh \ +--cleanup-policy "Compact" \ +--status "Active" \ +--partition-count 1 \ +--retention-time 24 + +echo ">>>>>> 2" + +az eventhubs eventhub create \ +-g pagopa-d-itn-observ-evh-rg \ +-n "connect-cluster-status" \ +--namespace-name pagopa-d-itn-observ-gpd-evh \ +--cleanup-policy "Compact" \ +--status "Active" \ +--partition-count 1 \ +--retention-time 24 + +echo ">>>>>> 3" + +az eventhubs eventhub create \ +-g pagopa-d-itn-observ-evh-rg \ +-n "connect-cluster-configs" \ +--namespace-name pagopa-d-itn-observ-gpd-evh \ +--cleanup-policy "Compact" \ +--status "Active" \ +--partition-count 1 \ +--retention-time 24 + +echo ">>>>>> 4" + +az eventhubs eventhub create \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd-ingestion.apd.payment_option" \ +--namespace-name pagopa-d-itn-observ-gpd-evh \ +--cleanup-policy "Compact" \ +--status "Active" \ +--partition-count 1 \ +--retention-time 24 + +echo ">>>>>> 4" + +az eventhubs eventhub create \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd-ingestion.apd.payment_option_metadata" \ +--namespace-name pagopa-d-itn-observ-gpd-evh \ +--cleanup-policy "Compact" \ +--status "Active" \ +--partition-count 1 \ +--retention-time 24 + +echo ">>>>>> 5" + +az eventhubs eventhub create \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd-ingestion.apd.payment_position" \ +--namespace-name pagopa-d-itn-observ-gpd-evh \ +--cleanup-policy "Compact" \ +--status "Active" \ +--partition-count 1 \ +--retention-time 24 + +echo ">>>>>> 6" + +az eventhubs eventhub create \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd-ingestion.apd.transfer" \ +--namespace-name pagopa-d-itn-observ-gpd-evh \ +--cleanup-policy "Compact" \ +--status "Active" \ +--partition-count 1 \ +--retention-time 24 + +echo ">>>>>> 7" + +az eventhubs eventhub create \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd-ingestion.apd.transfer_metadata" \ +--namespace-name pagopa-d-itn-observ-gpd-evh \ +--cleanup-policy "Compact" \ +--status "Active" \ +--partition-count 1 \ +--retention-time 24 diff --git a/src/domains/observability/gpd_evh_delete__az.sh b/src/domains/observability/gpd_evh_delete__az.sh new file mode 100644 index 0000000000..18208a3e57 --- /dev/null +++ b/src/domains/observability/gpd_evh_delete__az.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +set -e + +echo ">>>>>> 1" + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "connect-cluster-offsets" \ +--namespace-name pagopa-d-itn-observ-gpd-evh + +echo ">>>>>> 2" + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "connect-cluster-status" \ +--namespace-name pagopa-d-itn-observ-gpd-evh + +echo ">>>>>> 3" + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "connect-cluster-configs" \ +--namespace-name pagopa-d-itn-observ-gpd-evh + +echo ">>>>>> 4" + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd-ingestion.apd.payment_option" \ +--namespace-name pagopa-d-itn-observ-gpd-evh + +echo ">>>>>> 4" + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd-ingestion.apd.payment_option_metadata" \ +--namespace-name pagopa-d-itn-observ-gpd-evh + + +echo ">>>>>> 5" + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd-ingestion.apd.payment_position" \ +--namespace-name pagopa-d-itn-observ-gpd-evh + +echo ">>>>>> 6" + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd-ingestion.apd.transfer" \ +--namespace-name pagopa-d-itn-observ-gpd-evh + +echo ">>>>>> 7" + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd-ingestion.apd.transfer_metadata" \ +--namespace-name pagopa-d-itn-observ-gpd-evh From 3ab638caa238864f0845ba28309d5cb4cdabd7ea Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 4 Nov 2024 14:05:24 +0100 Subject: [PATCH 18/55] fix wait chk resources --- src/domains/gps-app/05_debezium_connect.tf | 4 +- src/domains/gps-app/yaml/kafka-connect.yaml | 3 +- .../observability/gpd_evh_create__az.sh | 9 +++++ .../observability/gpd_evh_delete__az.sh | 38 +++++++++++++++++++ 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index bba1836fa0..7278442ef1 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -129,7 +129,7 @@ resource "null_resource" "wait_kafka_connect" { kubectl_manifest.kafka_connect ] provisioner "local-exec" { - command = "while [ true ]; do STATUS=`kubectl -n gps get KafkaConnect -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Kafka Connect SUCCEEDED\" ; break ; else echo \"Kafka Connect INPROGRESS\"; sleep 3; fi ; done" + command = "while [ true ]; do STATUS=`kubectl -n gps get KafkaConnect -o json | jq -r '.items[] | select(.status).status | .conditions | any(.[]; .type == \"Ready\")' | uniq`; if [ \"$STATUS\" = \"true\" ]; then echo \"Kafka Connect SUCCEEDED\" ; break ; else echo \"Kafka Connect INPROGRESS\"; sleep 3; fi ; done" interpreter = ["/bin/bash", "-c"] } } @@ -147,7 +147,7 @@ resource "null_resource" "wait_postgres_connector" { kubectl_manifest.kafka_connect ] provisioner "local-exec" { - command = "while [ true ]; do STATUS=`kubectl -n gps get KafkaConnector -ojsonpath='{range .items[*]}{.status.health}'`; if [ \"$STATUS\" = \"green\" ]; then echo \"Postgres Connector SUCCEEDED\" ; break ; else echo \"Postgres Connector INPROGRESS\"; sleep 3; fi ; done" + command = "while [ true ]; do STATUS=`kubectl -n gps get KafkaConnector -o json | jq -r '.items[] | select(.status).status | .conditions | any(.[]; .type == \"Ready\")' | uniq`; if [ \"$STATUS\" = \"true\" ]; then echo \"Postgres Connector SUCCEEDED\" ; break ; else echo \"Postgres Connector INPROGRESS\"; sleep 3; fi ; done" interpreter = ["/bin/bash", "-c"] } } diff --git a/src/domains/gps-app/yaml/kafka-connect.yaml b/src/domains/gps-app/yaml/kafka-connect.yaml index fec54494c8..19ebd11041 100644 --- a/src/domains/gps-app/yaml/kafka-connect.yaml +++ b/src/domains/gps-app/yaml/kafka-connect.yaml @@ -37,7 +37,8 @@ spec: status.storage.replication.factor: 1 rest.advertised.host.name: connect offset.flush.interval.ms: 10000 - topic.creation.enable: false + topic.creation.enable: "false" + auto.create.topics.enable: "false" # https://debezium.io/documentation/reference/3.0/configuration/topic-auto-create-config.html#disabling-automatic-topic-creation-for-the-kafka-broker key.converter: org.apache.kafka.connect.json.JsonConverter value.converter: org.apache.kafka.connect.json.JsonConverter internal.key.converter: org.apache.kafka.connect.json.JsonConverter diff --git a/src/domains/observability/gpd_evh_create__az.sh b/src/domains/observability/gpd_evh_create__az.sh index 4492b72f74..f200ab1088 100644 --- a/src/domains/observability/gpd_evh_create__az.sh +++ b/src/domains/observability/gpd_evh_create__az.sh @@ -2,6 +2,11 @@ set -e +# ============================================================== +# config topics +# ============================================================== + + echo ">>>>>> 1" az eventhubs eventhub create \ @@ -35,6 +40,10 @@ az eventhubs eventhub create \ --partition-count 1 \ --retention-time 24 +# ============================================================== +# logical topics +# ============================================================== + echo ">>>>>> 4" az eventhubs eventhub create \ diff --git a/src/domains/observability/gpd_evh_delete__az.sh b/src/domains/observability/gpd_evh_delete__az.sh index 18208a3e57..cdecbe5a06 100644 --- a/src/domains/observability/gpd_evh_delete__az.sh +++ b/src/domains/observability/gpd_evh_delete__az.sh @@ -2,6 +2,11 @@ set -e +# ============================================================== +# config topics +# ============================================================== + + echo ">>>>>> 1" az eventhubs eventhub delete \ @@ -23,6 +28,10 @@ az eventhubs eventhub delete \ -n "connect-cluster-configs" \ --namespace-name pagopa-d-itn-observ-gpd-evh +# ============================================================== +# logical topics +# ============================================================== + echo ">>>>>> 4" az eventhubs eventhub delete \ @@ -58,3 +67,32 @@ az eventhubs eventhub delete \ -g pagopa-d-itn-observ-evh-rg \ -n "azcligpd-ingestion.apd.transfer_metadata" \ --namespace-name pagopa-d-itn-observ-gpd-evh + +# auto-create + +echo ">>>>>> ....." + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd.apd.payment_option" \ +--namespace-name pagopa-d-itn-observ-gpd-evh + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd.apd.payment_option_metadata" \ +--namespace-name pagopa-d-itn-observ-gpd-evh + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd.apd.payment_position" \ +--namespace-name pagopa-d-itn-observ-gpd-evh + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd.apd.transfer" \ +--namespace-name pagopa-d-itn-observ-gpd-evh + +az eventhubs eventhub delete \ +-g pagopa-d-itn-observ-evh-rg \ +-n "azcligpd.apd.transfer_metadata" \ +--namespace-name pagopa-d-itn-observ-gpd-evh From 87f516d362d7886efd5405b2d95761e83c682527 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 4 Nov 2024 15:09:07 +0100 Subject: [PATCH 19/55] fix --- src/aks-leonardo/.terraform.lock.hcl | 227 +++++++++++++++------------ src/aks-leonardo/03_aks_0.tf | 7 +- 2 files changed, 128 insertions(+), 106 deletions(-) diff --git a/src/aks-leonardo/.terraform.lock.hcl b/src/aks-leonardo/.terraform.lock.hcl index 5fab8d9bde..2f8601048a 100644 --- a/src/aks-leonardo/.terraform.lock.hcl +++ b/src/aks-leonardo/.terraform.lock.hcl @@ -2,143 +2,168 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/alekc/kubectl" { - version = "2.1.0" + version = "2.0.4" constraints = "~> 2.0" hashes = [ - "h1:fyE+ICPznpHaRAIT/GtIUdl7Z5MqBpXhnLH26+FlpT8=", - "zh:030d9aaaa251fb9f2b98640f343b1944a09924a3507340590552f5dfb037c1e2", - "zh:1a1672cd6a60d0a5296bd89d92b2113af9105ce933629c0195416013744db16f", - "zh:1cfc7bfbe6f145acd08fb52289f0fe4ed36f3a5e0d93f6b221e40236d164a5b2", - "zh:36e2620433b497f1538d84647e7041042bc43de9b3491febe5cb9ec0b47401b8", - "zh:5b301ff79f6b80869d6f5e54abdc63d7dde146af9b3c37340f7af922321cf316", - "zh:6f63ce78866dc3f5ea127825a70a11d53cb93f5dfa6187e8390592dd2f8857f9", - "zh:73e51fe86ec9263ab60507b3c811875074532613abf73154ab848fda181e078a", - "zh:8e65fe5b8465f25fadb4a7411981aeb307e2f482060b2642795fe371883efbb2", - "zh:91c07d9120687ba93f13af24f44cdf19d0c96429da90b384d10c4bf2bcf5725e", - "zh:c53cdefc0a25113e09bdf3c57a1c064d937b783fbcf9bb9228e9309d95294b9e", - "zh:c652849feab85900c881af20effaa26052bdadba5eaafefce9d09e15c8c6c32f", - "zh:c739f54428c0ad83f7031ae29d56c377026619912b814ba03ad37e92df558125", - "zh:d0cd843e29984889be06a61e0eebe6dccf669563f8130d1066f50552507db66f", - "zh:e9eb47fdda142d1f51cdd486ff46bf089a9c55ec93ac1c6d36d2e757ed217ee5", + "h1:TUeUq1UdVkHTxcgq7CJWWXBrc8VEQTufmgU18qDmfGE=", + "h1:mCz0lOwNsFCZEcFf7DBSe6b4hZgn5piiy0mZDwRGUIU=", + "zh:15c227886bac78c8b8827f85595648212574ec81febc39e1055e1a6bf048fe65", + "zh:2211ebeeb0918dbb3587d206e32adca9e1f343a93bbffcd37d8d99bf4d8dea9a", + "zh:2303836cdea12ece8dbe39c2d7d30a9378fd06e9c2ebda66cbe5e01cc096ee2e", + "zh:3687f69e531c70845682b214888a9959b93f2be3c2531801228a4b1965d59921", + "zh:4dd686b4c55e2eedd80464984c9bb736c2df7a96d9dd59a692d91d09173f5f64", + "zh:51e29c13a87e56867b4be0b0c68da874149bf6d4014d7259b62d91162142c1bd", + "zh:5d9d99260f2adfb8867068a3d7644336d57cfa7710062c5221dcbb5a7ec90c7d", + "zh:901c19d73da6688437b19a85e3cd60e8f2090c84699e108b31953bb87f6d3141", + "zh:9547743606a36fa6b6748c5e2e1959b6f185730a1da53a3c351cfa0d8c096687", + "zh:9772a30704e69b54de5a332858a39591f52286121cffcba702346830b1c6e362", + "zh:b44792f99d7c90b9a364dd922f861e459ae1b1edc039f6b3078549021fec4511", + "zh:b5eb871ed2e39b9236dce06170b1fd5dda29f3c1d53f8e08285ccb9a4f574201", + "zh:e8bb4c3d9f680977b560e9dec24662650f790259b2c1311ee07a72157f6492b3", + "zh:f4772cfa0f9c73fdef008bb917cd268620009dc7ff270a4d819125c642b5acce", ] } provider "registry.terraform.io/hashicorp/azuread" { - version = "3.0.2" - constraints = "<= 3.0.2" + version = "2.47.0" + constraints = "<= 2.47.0" hashes = [ - "h1:HNrx7UJEDY5Kbx/r1LRQDWnziqvB6x3IU+pEA8Vq7dw=", - "zh:16e724b80a9004c7978c30f69a73c98ff63eb8a03937dd44c2a8f0ea0438b7a3", + "h1:8J74v92UvtqVNucugAtB+Sd44oTgnhfct+Xf8ObOZug=", + "h1:KB9BNRNStbdsfdRmVXUwXtN77qgX5VjBy2UALcqp218=", + "h1:g8+gBFM4QVOEQFqAEs5pR6iXpbGvgPvcEi1evHwziyw=", + "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", + "h1:zYMGokLn44KSWir7Nr4t8lEAPMB6JuXd2LlP2Ac2tMY=", + "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", - "zh:2bbbf13713ca4767267b889471c9fc14a56a8fdf5d1013da3ca78667e3caec64", - "zh:409ccb05431d643a079da082d89db2d95d6afed4769997ac537c8b7de3bff867", - "zh:53e4bca0f5d015380f7f524f36344afe6211ccaf614bfc69af73ca64a9f47d6c", - "zh:5780be2c1981d090604d7fa4cef675462f17f40e7f3dc501a031488e87a35b8f", - "zh:850e61a1b3e64c752c418526ccf48653514c861b36f5feb631619f906f7e99a0", - "zh:8c3565bfcea006a734149cc080452a9daf7d2a9d5362eb7e0a088b6c0d7f0f03", - "zh:908b9e6ad49d5d21173ecefc7924902047611be93bbf8e7d021aa9563358396f", - "zh:a2a79765c029bc58966eff61cb6e9b0ee14d2ac52b0a22fc7dfa35c9a49af669", - "zh:c7f56cbe8743e9ba81fce871bc97d9c07abe86770d9ee7ffefbf3882a61ba89a", - "zh:d4dba80e33421b30d81c62611fb7fc62ad39afecc6484436e635913cd8553e67", + "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", + "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", + "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", + "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", + "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", + "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", + "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", + "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", + "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", + "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", ] } provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.116.0" - constraints = "~> 3.30, ~> 3.100, <= 3.116.0" + version = "3.105.0" + constraints = "~> 3.30, <= 3.105.0" hashes = [ - "h1:2QbjtN4oMXzdA++Nvrj/wSmWZTPgXKOSFGGQCLEMrb4=", - "zh:02b6606aff025fc2a962b3e568e000300abe959adac987183c24dac8eb057f4d", - "zh:2a23a8ce24ff9e885925ffee0c3ea7eadba7a702541d05869275778aa47bdea7", - "zh:57d10746384baeca4d5c56e88872727cdc150f437b8c5e14f0542127f7475e24", - "zh:59e3ebde1a2e1e094c671e179f231ead60684390dbf02d2b1b7fe67a228daa1a", - "zh:5f1f5c7d09efa2ee8ddf21bd9efbbf8286f6e90047556bef305c062fa0ac5880", - "zh:a40646aee3c9907276dab926e6123a8d70b1e56174836d4c59a9992034f88d70", - "zh:c21d40461bc5836cf56ad3d93d2fc47f61138574a55e972ad5ff1cb73bab66dc", - "zh:c56fb91a5ae66153ba0f737a26da1b3d4f88fdef7d41c63e06c5772d93b26953", - "zh:d1e60e85f51d12fc150aeab8e31d3f18f859c32f927f99deb5b74cb1e10087aa", - "zh:ed35e727e7d79e687cd3d148f52b442961ede286e7c5b4da1dcd9f0128009466", + "h1:MK83TecMdabDD+HjbxdTt3emXp8G6djLj7KvvUGstM0=", + "h1:OtWRTAMNOruOmwVB72QSGXC5IIGGQcHwEqnCCmsGbGM=", + "h1:SOC7EdvKd5YowghQvb6hu209F1PQqtb8LulbQkxOZQQ=", + "h1:tEDW5rEALglcH1JRy31z6AzDULECYrAZOD24B4mqry8=", + "h1:zWkzhP2fx0WQIAUp6Amk/We3WNcbtiWagpKF5PJP5+M=", + "zh:2f81bca6a3bf3d37604bf99fdb2c77d6118520aa379ab65fd28e6b76bed399cd", + "zh:3578eb79d175af9544b0dc543124d551c0fed4c48f51773ee17e1dc62e22833a", + "zh:377dbb56caea3fa1e6a6599193b55c8594204c40c054fc2ace4f576fdfe750a2", + "zh:3d1cf01929cb213ff9a0f9753e35699bf13f60f7f0f15b38f1b216fa2cbb5f72", + "zh:481376d79224a0e4aebc6e39dee10de3cc14efd1c7c58b6d74c538e356cf4bb2", + "zh:625119aec0d24ff693c589d802b7983ffce3fcf1e9c3351396af02799dd176ca", + "zh:65981e62a6e9eb8a39dd332632617e8c929dcce6af48d3829f590f5c0f14362f", + "zh:72db82697f4e694c21defa8d0efb22f71d373676d078d71d567e8b4d9a134df7", + "zh:a0fa43cf78716ff98eccd7506b017c5c487034d9d9cce88c1accdba9314a4822", + "zh:b073f60b68b0102128815251dd895ec7f24bddec84a1b725fc0777de8a78dc7e", + "zh:b601e509eb9735756b6b7ccacc15d6333769a7bb2f8ac8c394e39b29cfc6ee55", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", - "zh:f6d2a4e7c58f44e7d04a4a9c73f35ed452f412c97c85def68c4b52814cbe03ab", ] } provider "registry.terraform.io/hashicorp/external" { - version = "2.3.4" - constraints = "<= 2.3.4" + version = "2.3.3" + constraints = "<= 2.3.3" hashes = [ - "h1:U6W8rgrdmR2pZ2cicFoGOSQ4GXuIf/4EK7s0vTJN7is=", - "zh:037fd82cd86227359bc010672cd174235e2d337601d4686f526d0f53c87447cb", - "zh:0ea1db63d6173d01f2fa8eb8989f0809a55135a0d8d424b08ba5dabad73095fa", - "zh:17a4d0a306566f2e45778fbac48744b6fd9c958aaa359e79f144c6358cb93af0", - "zh:298e5408ab17fd2e90d2cd6d406c6d02344fe610de5b7dae943a58b958e76691", - "zh:38ecfd29ee0785fd93164812dcbe0664ebbe5417473f3b2658087ca5a0286ecb", - "zh:59f6a6f31acf66f4ea3667a555a70eba5d406c6e6d93c2c641b81d63261eeace", + "h1:/x65slrvO8YG5MKxE2DaU5udEbUxBu3BgEiO7EEM9bQ=", + "h1:H+3QlVPs/7CDa3I4KU/a23wYeGeJxeBlgvR7bfK1t1w=", + "h1:Qi72kOSrEYgEt5itloFhDfmiFZ7wnRy3+F74XsRuUOw=", + "h1:Up2xaIhiNYomK8Lhe29U2FcojpbRWZYDtSeS03OhI94=", + "h1:gShzO1rJtADK9tDZMvMgjciVAzsBh39LNjtThCwX1Hg=", + "zh:03d81462f9578ec91ce8e26f887e34151eda0e100f57e9772dbea86363588239", + "zh:37ec2a20f6a3ec3a0fd95d3f3de26da6cb9534b30488bc45723e118a0911c0d8", + "zh:4eb5b119179539f2749ce9de0e1b9629d025990f062f4f4dddc161562bb89d37", + "zh:5a31bb58414f41bee5e09b939012df5b88654120b0238a89dfd6691ba197619a", + "zh:6221a05e52a6a2d4f520ffe7cbc741f4f6080e0855061b0ed54e8be4a84eb9b7", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:ad0279dfd09d713db0c18469f585e58d04748ca72d9ada83883492e0dd13bd58", - "zh:c69f66fd21f5e2c8ecf7ca68d9091c40f19ad913aef21e3ce23836e91b8cbb5f", - "zh:d4a56f8c48aa86fc8e0c233d56850f5783f322d6336f3bf1916e293246b6b5d4", - "zh:f2b394ebd4af33f343835517e80fc876f79361f4688220833bc3c77655dd2202", - "zh:f31982f29f12834e5d21e010856eddd19d59cd8f449adf470655bfd19354377e", + "zh:8bb068496b4679bef625e4710d9f3432e301c3a56602271f04e60eadf7f8a94c", + "zh:94742aa5378bab626ce34f79bcef6a373e4f86ea7a8b762e9f71270a899e0d00", + "zh:a485831b5a525cd8f40e8982fa37da40ff70b1ae092c8b755fcde123f0b1238d", + "zh:a647ff16d071eabcabd87ea8183eb90a775a0294ddd735d742075d62fff09193", + "zh:b74710c5954aaa3faf262c18d36a8c2407862d9f842c63e7fa92fa4de3d29df6", + "zh:fa73d83edc92af2e551857594c2232ba6a9e3603ad34b0a5940865202c08d8d7", ] } provider "registry.terraform.io/hashicorp/helm" { - version = "2.16.0" - constraints = ">= 2.0.0, ~> 2.12, <= 2.16.0" + version = "2.12.1" + constraints = ">= 2.0.0, ~> 2.12, <= 2.12.1" hashes = [ - "h1:uJs402IoDa/7+AnBQZC1txmO0jY4v9W1TMHAvRaCZkY=", - "zh:0fa970817bab7a8411ff443d51004dc2974c0ef4aad082a514f8b56559db3113", - "zh:333b9ac02fcbf9dcf4825dc1e4fc373ef4571b1dd00b79f5c8ea24e1c79992f0", - "zh:792e1e9c409dd76e3eabf3b0c0a6b5a3c3ef42adfc578f7899def46a81e994ef", - "zh:8eca4a52d43ca97d944a8c5d0f2ee60bcbefcb3ccee51d5620bde9047b8ea9c7", - "zh:90969e6a0f7127b0cb75c8790f63f4d050576ffe9bd722887a11d885430624cd", - "zh:a9d72fb106f16ab4f68c779a2c59124929cbc1cb0dbc47ed5ef380c6205f70bb", - "zh:c28bc1a2c0f8f11626baf905a888b2600663ba8dbb33ce4203efcafa16c77fc5", - "zh:c5d6c72a8c5513ff868209ceda9e6000723b02d21811d05909d26614784d4db6", - "zh:d105d40b1a217120332f65a93b24470d18e355868bfa99f0cdeeff5869cff9fb", - "zh:e6c78637c8c6081b8817f61658de8d0163b92157336ac3236cf183b5834f9487", - "zh:edef68729e4f263df3a6737fc73b14e1ee952b800d72d0c6f2cb524bc1ad7ec8", + "h1:7wfYOAeSEchHB8idNl+2jf+OkFi9zFSOLWkEZFuTCik=", + "h1:aBfcqM4cbywa7TAxfT1YoFS+Cst9waerlm4XErFmJlk=", + "h1:sgYI7lwGqJqPopY3NGmhb1eQ0YbH8PIXaAZAmnJrAvw=", + "h1:sjzfyNQAjtF9zXHxB67geryjGkHaPDMMVw9iqPP5pkE=", + "h1:xwHVa6ab/XVfDrZ3h35OzLJ6g0Zte4VAvSnyKw3f9AI=", + "zh:1d623fb1662703f2feb7860e3c795d849c77640eecbc5a776784d08807b15004", + "zh:253a5bc62ba2c4314875139e3fbd2feaad5ef6b0fb420302a474ab49e8e51a38", + "zh:282358f4ad4f20d0ccaab670b8645228bfad1c03ac0d0df5889f0aea8aeac01a", + "zh:4fd06af3091a382b3f0d8f0a60880f59640d2b6d9d6a31f9a873c6f1bde1ec50", + "zh:6816976b1830f5629ae279569175e88b497abbbac30ee809948a1f923c67a80d", + "zh:7d82c4150cdbf48cfeec867be94c7b9bd7682474d4df0ebb7e24e148f964844f", + "zh:83f062049eea2513118a4c6054fb06c8600bac96196f25aed2cc21898ec86e93", + "zh:a79eec0cf4c08fca79e44033ec6e470f25ff23c3e2c7f9bc707ed7771c1072c0", + "zh:b2b2d904b2821a6e579910320605bc478bbef063579a23fbfdd6fcb5871b81f8", + "zh:e91177ca06a15487fc570cb81ecef6359aa399459ea2aa7c4f7367ba86f6fcad", + "zh:e976bcb82996fc4968f8382bbcb6673efb1f586bf92074058a232028d97825b1", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } provider "registry.terraform.io/hashicorp/kubernetes" { - version = "2.33.0" - constraints = "~> 2.27, <= 2.33.0" + version = "2.27.0" + constraints = "~> 2.27, <= 2.27.0" hashes = [ - "h1:44s6P+u1FUHyEclCAyko9UL+PB73rGp+REnCML3hyzg=", - "zh:255b35790b706d405e987750190658dcaefb663741b96803a9529ba5d7435329", - "zh:362feba1aa820a8e02869ec71d1a08e87243dbce43671dc0995fa6c5a2fafa1d", - "zh:39332abcf75b5dd9c78c79c7c0c094f7d4ca908d1b76bbd2aae67e8e3516710c", - "zh:3e8e7f758bb09a9b5b613c8866e77541f8f00b521070cc86bc095ce61f010baf", - "zh:427883b889b9c36630c3eec4d5c07bc4ae12cc0d358fc17ea42a8049bf8d5275", - "zh:69bfc4ed067a5e4844db1a1809343652ff239aa0a8da089b1671524c44e8740a", - "zh:6b9f731062b945c5020e0930ed9a1b1b50afd2caf751f0e70a282d165c970979", - "zh:6faf9ec006af7ee7014a9c3251d65b701792abb823f149b0b7e4ac4433848201", - "zh:b706f76d695104a47682ee6ab842870f9c70a680f979fa9e7efe34278c0831bc", - "zh:b9bca48de2c92f57389ed58dd2fac564deaccd79a92cafd08edeed3ba6b91d4d", - "zh:bbd3336dbee5aed9880f98e36fb8340e0c6d8f0399a05787521af599ccb3dac4", + "h1:/3kLyOR2jTaWS1MKso4xAztrocGBMxi8yVadWiqSWOg=", + "h1:GzU0FzYAT/+IgAhnSBcFH3bT+4I5N6oSga6iZgNJAus=", + "h1:TrlG/sofnDv8kAbzKOD5pIPeUiI5VQY61NuWH+cItDw=", + "h1:WuU4rl7szPJr9Nfu5OoQGF84k8yQf+gmS9zU2eZuxcc=", + "h1:w9ENsSqT/3Oj/yt4GcudG202ehSD2Ls5gwqOLoKrBUQ=", + "zh:3bdba30ae67c55dc7e9a317ac0da3b208ea7926fe9c2f0ae6587ee88dcc58d1f", + "zh:3f35138a831c00b188d2ffee27111dd0cf59afad2dd5653ed9e67d59646de12c", + "zh:64066d18f6ae9a316c2bc840ef3e641d7ab94e1ea3a41d12523e77345ad442ef", + "zh:653063d44b44881af3a480f7f8eaa94fa300e0229df2072d30f606bddcc9f025", + "zh:87f306e37efb61d13efa6da53a1e45e97e5996ebc0568b1caf8c3c5e54c05809", + "zh:8c428b9708f9634391e52300218771eab3fe942bb1295d8c0ad50ca4b33db3d9", + "zh:a44e87119a0337ded15479851786a13f412b413d9a463ba550d1210249206b0f", + "zh:aa2c4d110b0de6ef997c0d45f3f23f8a98f5530753095d6eff439a6d91a8ea31", + "zh:eb15ed8781ac6a0dec2f7d03cf090e23cfa05e3225806c6231ff2c574662fd63", + "zh:eb81c563f93bd3303f9620d11cd49f21f3f89ac3475c6d3e821b239feb9c217d", + "zh:f1a344a7f16131123577e4ec994d04a34ea458ec16c1ccac53fe7946bd817b18", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } provider "registry.terraform.io/hashicorp/null" { - version = "3.2.3" - constraints = "~> 3.2, <= 3.2.3" + version = "3.2.2" + constraints = "~> 3.2, <= 3.2.2" hashes = [ - "h1:nKUqWEza6Lcv3xRlzeiRQrHtqvzX1BhIzjaOVXRYQXQ=", - "zh:22d062e5278d872fe7aed834f5577ba0a5afe34a3bdac2b81f828d8d3e6706d2", - "zh:23dead00493ad863729495dc212fd6c29b8293e707b055ce5ba21ee453ce552d", - "zh:28299accf21763ca1ca144d8f660688d7c2ad0b105b7202554ca60b02a3856d3", - "zh:55c9e8a9ac25a7652df8c51a8a9a422bd67d784061b1de2dc9fe6c3cb4e77f2f", - "zh:756586535d11698a216291c06b9ed8a5cc6a4ec43eee1ee09ecd5c6a9e297ac1", + "h1:Gef5VGfobY5uokA5nV/zFvWeMNR2Pmq79DH94QnNZPM=", + "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", + "h1:m467k2tZ9cdFFgHW7LPBK2GLPH43LC6wc3ppxr8yvoE=", + "h1:vWAsYRd7MjYr3adj8BVKRohVfHpWQdvkIwUQ2Jf5FVM=", + "h1:zT1ZbegaAYHwQa+QwIFugArWikRJI9dqohj8xb0GY88=", + "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", + "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", + "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", + "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", + "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", + "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", + "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:9d5eea62fdb587eeb96a8c4d782459f4e6b73baeece4d04b4a40e44faaee9301", - "zh:a6355f596a3fb8fc85c2fb054ab14e722991533f87f928e7169a486462c74670", - "zh:b5a65a789cff4ada58a5baffc76cb9767dc26ec6b45c00d2ec8b1b027f6db4ed", - "zh:db5ab669cf11d0e9f81dc380a6fdfcac437aea3d69109c7aef1a5426639d2d65", - "zh:de655d251c470197bcbb5ac45d289595295acb8f829f6c781d4a75c8c8b7c7dd", - "zh:f5c68199f2e6076bce92a12230434782bf768103a427e9bb9abee99b116af7b5", + "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", + "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", + "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", + "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", ] -} +} \ No newline at end of file diff --git a/src/aks-leonardo/03_aks_0.tf b/src/aks-leonardo/03_aks_0.tf index eb01a03159..20bad89262 100644 --- a/src/aks-leonardo/03_aks_0.tf +++ b/src/aks-leonardo/03_aks_0.tf @@ -6,7 +6,7 @@ resource "azurerm_resource_group" "rg_aks" { } module "aks_leonardo" { - source = "./.terraform/modules/__v3__/kubernetes_cluster" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_cluster?ref=v8.20.1" name = local.aks_cluster_name location = var.location @@ -31,9 +31,6 @@ module "aks_leonardo" { system_node_pool_node_labels = var.aks_system_node_pool.node_labels system_node_pool_tags = var.aks_system_node_pool.node_tags - workload_identity_enabled = var.env_short == "d" ? true : false - oidc_issuer_enabled = var.env_short == "d" ? true : false - # # ☁️ Network # @@ -145,4 +142,4 @@ resource "azurerm_role_assignment" "aks_to_acr" { scope = data.azurerm_container_registry.acr.id role_definition_name = "AcrPull" principal_id = module.aks_leonardo.kubelet_identity_id -} +} \ No newline at end of file From a78935ecc744e4821f2f64ba39221c4e29f284c0 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 4 Nov 2024 15:11:30 +0100 Subject: [PATCH 20/55] fix --- src/aks-leonardo/.terraform.lock.hcl | 2 +- src/aks-leonardo/03_aks_0.tf | 2 +- src/aks-leonardo/03_aks_storage.tf | 2 +- src/aks-leonardo/03_monitoring.tf | 5 ++--- src/aks-leonardo/99_main.tf | 16 ++++++---------- 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/aks-leonardo/.terraform.lock.hcl b/src/aks-leonardo/.terraform.lock.hcl index 2f8601048a..d2daf422eb 100644 --- a/src/aks-leonardo/.terraform.lock.hcl +++ b/src/aks-leonardo/.terraform.lock.hcl @@ -166,4 +166,4 @@ provider "registry.terraform.io/hashicorp/null" { "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", ] -} \ No newline at end of file +} diff --git a/src/aks-leonardo/03_aks_0.tf b/src/aks-leonardo/03_aks_0.tf index 20bad89262..62397c2e95 100644 --- a/src/aks-leonardo/03_aks_0.tf +++ b/src/aks-leonardo/03_aks_0.tf @@ -142,4 +142,4 @@ resource "azurerm_role_assignment" "aks_to_acr" { scope = data.azurerm_container_registry.acr.id role_definition_name = "AcrPull" principal_id = module.aks_leonardo.kubelet_identity_id -} \ No newline at end of file +} diff --git a/src/aks-leonardo/03_aks_storage.tf b/src/aks-leonardo/03_aks_storage.tf index 9d0e78d054..93ca67ec3a 100644 --- a/src/aks-leonardo/03_aks_storage.tf +++ b/src/aks-leonardo/03_aks_storage.tf @@ -1,3 +1,3 @@ module "aks_storage_class" { - source = "./.terraform/modules/__v3__/kubernetes_storage_class" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_storage_class?ref=v8.17.1" } diff --git a/src/aks-leonardo/03_monitoring.tf b/src/aks-leonardo/03_monitoring.tf index 3aab21fd9f..932d7bff1d 100644 --- a/src/aks-leonardo/03_monitoring.tf +++ b/src/aks-leonardo/03_monitoring.tf @@ -5,7 +5,7 @@ resource "kubernetes_namespace" "monitoring" { } module "aks_prometheus_install" { - source = "./.terraform/modules/__v3__/kubernetes_prometheus_install" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_prometheus_install?ref=v8.17.1" prometheus_namespace = kubernetes_namespace.monitoring.metadata[0].name storage_class_name = "default-zrs" @@ -13,7 +13,7 @@ module "aks_prometheus_install" { module "elastic_agent" { - source = "./.terraform/modules/__v3__/elastic_agent" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//elastic_agent?ref=v8.50.0" es_host = var.env == "p" ? "https://weu${var.env}.kibana.internal.platform.pagopa.it:443/elastic" : "https://weu${var.env}.kibana.internal.${var.env}.platform.pagopa.it:443/elastic" @@ -28,4 +28,3 @@ module "elastic_agent" { } // TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops - diff --git a/src/aks-leonardo/99_main.tf b/src/aks-leonardo/99_main.tf index dc6789fcab..be793930c3 100644 --- a/src/aks-leonardo/99_main.tf +++ b/src/aks-leonardo/99_main.tf @@ -3,27 +3,27 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "<= 3.116.0" + version = "<= 3.105.0" } azuread = { source = "hashicorp/azuread" - version = "<= 3.0.2" + version = "<= 2.47.0" } external = { source = "hashicorp/external" - version = "<= 2.3.4" + version = "<= 2.3.3" } kubernetes = { source = "hashicorp/kubernetes" - version = "<= 2.33.0" + version = "<= 2.27.0" } helm = { source = "hashicorp/helm" - version = "<= 2.16.0" + version = "<= 2.12.1" } null = { source = "hashicorp/null" - version = "<= 3.2.3" + version = "<= 3.2.2" } } @@ -51,7 +51,3 @@ provider "helm" { config_path = "${var.k8s_kube_config_path_prefix}/config-${local.aks_cluster_name}" } } - -module "__v3__" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" -} From e8e94b6cbd3b902a2c538c038f5be528818cf34c Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 4 Nov 2024 15:11:54 +0100 Subject: [PATCH 21/55] fix --- src/aks-leonardo/03_monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-leonardo/03_monitoring.tf b/src/aks-leonardo/03_monitoring.tf index 932d7bff1d..9803c321e0 100644 --- a/src/aks-leonardo/03_monitoring.tf +++ b/src/aks-leonardo/03_monitoring.tf @@ -27,4 +27,4 @@ module "elastic_agent" { } -// TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops +// TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops \ No newline at end of file From 47429c906fe0882ab56714b55888b8ed1b96450e Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 4 Nov 2024 15:12:21 +0100 Subject: [PATCH 22/55] fix --- src/aks-leonardo/03_monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-leonardo/03_monitoring.tf b/src/aks-leonardo/03_monitoring.tf index 9803c321e0..932d7bff1d 100644 --- a/src/aks-leonardo/03_monitoring.tf +++ b/src/aks-leonardo/03_monitoring.tf @@ -27,4 +27,4 @@ module "elastic_agent" { } -// TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops \ No newline at end of file +// TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops From 8172cb90b613f3b14de9a6085d2ab084176a762e Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 4 Nov 2024 15:12:56 +0100 Subject: [PATCH 23/55] fix --- src/aks-leonardo/03_monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-leonardo/03_monitoring.tf b/src/aks-leonardo/03_monitoring.tf index 932d7bff1d..9803c321e0 100644 --- a/src/aks-leonardo/03_monitoring.tf +++ b/src/aks-leonardo/03_monitoring.tf @@ -27,4 +27,4 @@ module "elastic_agent" { } -// TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops +// TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops \ No newline at end of file From 5beb9e55677bb656a3ac5cee0525bb77ca667f2a Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 4 Nov 2024 15:14:17 +0100 Subject: [PATCH 24/55] fix --- src/aks-leonardo/03_monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-leonardo/03_monitoring.tf b/src/aks-leonardo/03_monitoring.tf index 9803c321e0..0d36fee2d4 100644 --- a/src/aks-leonardo/03_monitoring.tf +++ b/src/aks-leonardo/03_monitoring.tf @@ -27,4 +27,4 @@ module "elastic_agent" { } -// TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops \ No newline at end of file +// TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops \ No newline at end of file From 95369519b0fda150845a9c97ea3b51fc985a4b8d Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 4 Nov 2024 15:14:49 +0100 Subject: [PATCH 25/55] fix --- src/aks-leonardo/03_monitoring.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aks-leonardo/03_monitoring.tf b/src/aks-leonardo/03_monitoring.tf index 0d36fee2d4..932d7bff1d 100644 --- a/src/aks-leonardo/03_monitoring.tf +++ b/src/aks-leonardo/03_monitoring.tf @@ -27,4 +27,4 @@ module "elastic_agent" { } -// TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops \ No newline at end of file +// TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops From 443005c7e3d5a52c43475c877f6d1eb8ad663f48 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 4 Nov 2024 15:17:31 +0100 Subject: [PATCH 26/55] fix --- src/aks-leonardo/03_monitoring.tf | 1 + .../paymentoptions-app/.terraform.lock.hcl | 102 +++++++++++++++++ .../paymentoptions-app/02_namespace.tf | 28 ++--- .../03_serviceaccounts_azure_devops.tf | 2 +- .../04_apim_payment_options.tf | 2 +- .../04_apim_payment_options_mock.tf | 4 +- .../05_aks_middleware_tools.tf | 65 ++++++----- src/domains/paymentoptions-app/99_main.tf | 15 +-- .../paymentoptions-common/.terraform.lock.hcl | 65 +++++++++++ .../paymentoptions-common/03_eventhub.tf | 4 +- .../10_github_identity.tf | 6 +- src/domains/paymentoptions-common/99_main.tf | 10 +- .../.terraform.lock.hcl | 107 ++++++++++++++++++ .../paymentoptions-secrets/01_keyvault.tf | 4 +- src/domains/paymentoptions-secrets/99_main.tf | 14 +-- 15 files changed, 341 insertions(+), 88 deletions(-) create mode 100644 src/domains/paymentoptions-app/.terraform.lock.hcl create mode 100644 src/domains/paymentoptions-common/.terraform.lock.hcl create mode 100644 src/domains/paymentoptions-secrets/.terraform.lock.hcl diff --git a/src/aks-leonardo/03_monitoring.tf b/src/aks-leonardo/03_monitoring.tf index 932d7bff1d..cae3682a43 100644 --- a/src/aks-leonardo/03_monitoring.tf +++ b/src/aks-leonardo/03_monitoring.tf @@ -28,3 +28,4 @@ module "elastic_agent" { } // TODO mettere nel kv il secret quickstart-es-elastic-user tramite sops + diff --git a/src/domains/paymentoptions-app/.terraform.lock.hcl b/src/domains/paymentoptions-app/.terraform.lock.hcl new file mode 100644 index 0000000000..97cef8e563 --- /dev/null +++ b/src/domains/paymentoptions-app/.terraform.lock.hcl @@ -0,0 +1,102 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "2.47.0" + constraints = "<= 2.47.0" + hashes = [ + "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", + "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", + "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", + "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", + "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", + "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", + "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", + "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", + "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", + "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", + "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.97.1" + constraints = ">= 3.30.0, ~> 3.30, <= 3.97.1, <= 3.106.0" + hashes = [ + "h1:LtwGbd4HEb5QCXmdxSvTjPSh8/Gp8eAQMYfiAKaubV4=", + "zh:15171efcc3aa3a37748c502c493cb16ecff603b81ada4499a843574976bac524", + "zh:2ca6c13a4a96f67763ecced0015c7b101ee02d54ea54b28a8df4ae06468071b1", + "zh:2e3c77dbfd8f760132ecef2d6117e939cbea26b96aba5e4d926e7f7f0f7afe72", + "zh:4bc346eece1622be93c73801d8256502b11fd7c2e7f7cea12d048bb9fc9fe900", + "zh:4f1042942ed8d0433680a367527289459d43b0894a51eaba83ac414e80d5187f", + "zh:63e674c31482ae3579ea84daf5b1ba066ce40cb23475f54e17b6b131320a1bec", + "zh:8327148766dcb7a174673729a832c8095d7e137d0e6c7e2a9a01da48b8b73fbe", + "zh:851b3ae417059a80c7813e7f0063298a590a42f056004f2c2558ea14061c207e", + "zh:ac081b48907139c121a422ae9b1f40fc72c6aaaeb05cbdbf848102a6a5f426f4", + "zh:dc1d663df2d95e4ba91070ceb20d3560b6ea5c465d39c57a5979319302643e41", + "zh:ed26457367cbbb94237e935d297cb31b5687f9abf697377da0ee46974480db9b", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.12.1" + constraints = "~> 2.12, <= 2.12.1" + hashes = [ + "h1:7wfYOAeSEchHB8idNl+2jf+OkFi9zFSOLWkEZFuTCik=", + "zh:1d623fb1662703f2feb7860e3c795d849c77640eecbc5a776784d08807b15004", + "zh:253a5bc62ba2c4314875139e3fbd2feaad5ef6b0fb420302a474ab49e8e51a38", + "zh:282358f4ad4f20d0ccaab670b8645228bfad1c03ac0d0df5889f0aea8aeac01a", + "zh:4fd06af3091a382b3f0d8f0a60880f59640d2b6d9d6a31f9a873c6f1bde1ec50", + "zh:6816976b1830f5629ae279569175e88b497abbbac30ee809948a1f923c67a80d", + "zh:7d82c4150cdbf48cfeec867be94c7b9bd7682474d4df0ebb7e24e148f964844f", + "zh:83f062049eea2513118a4c6054fb06c8600bac96196f25aed2cc21898ec86e93", + "zh:a79eec0cf4c08fca79e44033ec6e470f25ff23c3e2c7f9bc707ed7771c1072c0", + "zh:b2b2d904b2821a6e579910320605bc478bbef063579a23fbfdd6fcb5871b81f8", + "zh:e91177ca06a15487fc570cb81ecef6359aa399459ea2aa7c4f7367ba86f6fcad", + "zh:e976bcb82996fc4968f8382bbcb6673efb1f586bf92074058a232028d97825b1", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.29.0" + constraints = "~> 2.27, <= 2.29.0" + hashes = [ + "h1:Igs0JTtmzn5q7RHqrvrTMCD/DCSLPMinvUnhYZ2oITw=", + "zh:3edd5dc319b95fe94e61b82d10c1ce7fb53a2f21b067ddb742f2d7d0d19dd113", + "zh:4b9096e6d0cfa0efd4c89270e3d25fea49db570e2cfbe49c5d1de085a15f2578", + "zh:5397573838bcb8844248c8d6ac93cca7f39a0b707ac3ce7a7b306c50c261c195", + "zh:5d635370720d356b7bcb5756ca28de3275ca32ca1ef0201414caecd3a14759ac", + "zh:71a52280408f3fb0ff1866a9ab8059b0d9bde5481869658798e0773461f22eff", + "zh:748663ef0248d2d95f5dea2974332432a395165657856878c5dc6f000b37cc25", + "zh:7fbc1e084bbbb51e31afd3df0c77e833ae59e88cf42b9e2c17b0b1a1e3894723", + "zh:ae89b4be473b446270fa24dc1ef51b0cc4c2a528d9838ec15246d28bac165df3", + "zh:b6433970d680a0cc9898f915224508b5ece86ae4418372fa6bebd2a9d344f226", + "zh:bf871955cf49015e6a0433e814a22a109c1537a775b8b5dc7b37ad05c324904a", + "zh:c16fac91b2197b443a191d98cf37424feed550387ab11bd1427bde819722005e", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.1" + constraints = "~> 3.2, <= 3.2.1" + hashes = [ + "h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=", + "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", + "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", + "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", + "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", + "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", + "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", + "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", + "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", + "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", + "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", + ] +} diff --git a/src/domains/paymentoptions-app/02_namespace.tf b/src/domains/paymentoptions-app/02_namespace.tf index fe61498c87..a62b8cc66c 100644 --- a/src/domains/paymentoptions-app/02_namespace.tf +++ b/src/domains/paymentoptions-app/02_namespace.tf @@ -4,25 +4,17 @@ resource "kubernetes_namespace" "namespace" { } } -module "workload_identity" { - source = "./.terraform/modules/__v3__/kubernetes_workload_identity_init" +module "pod_identity" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_pod_identity?ref=v8.18.0" - workload_identity_name_prefix = "${var.domain}-workload-identity" - workload_identity_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name - workload_identity_location = var.location -} - -module "workload_identity" { - source = "./.terraform/modules/__v3__/kubernetes_workload_identity_configuration" + resource_group_name = local.aks_resource_group_name + location = var.location + tenant_id = data.azurerm_subscription.current.tenant_id + cluster_name = local.aks_name - workload_identity_name_prefix = "${var.domain}-poc" - workload_identity_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name - aks_name = data.azurerm_kubernetes_cluster.aks.name - aks_resource_group_name = data.azurerm_kubernetes_cluster.aks.resource_group_name - namespace = var.domain + identity_name = "${kubernetes_namespace.namespace.metadata[0].name}-pod-identity" + namespace = kubernetes_namespace.namespace.metadata[0].name + key_vault_id = data.azurerm_key_vault.kv.id - key_vault_id = data.azurerm_key_vault.kv.id - key_vault_certificate_permissions = ["Get"] - key_vault_key_permissions = ["Get"] - key_vault_secret_permissions = ["Get"] + secret_permissions = ["Get"] } diff --git a/src/domains/paymentoptions-app/03_serviceaccounts_azure_devops.tf b/src/domains/paymentoptions-app/03_serviceaccounts_azure_devops.tf index cfdc7c2574..f24964a97d 100644 --- a/src/domains/paymentoptions-app/03_serviceaccounts_azure_devops.tf +++ b/src/domains/paymentoptions-app/03_serviceaccounts_azure_devops.tf @@ -5,7 +5,7 @@ resource "kubernetes_namespace" "namespace_system" { } module "kubernetes_service_account" { - source = "./.terraform/modules/__v3__/kubernetes_service_account" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_service_account?ref=v8.18.0" name = "azure-devops" namespace = "${var.domain}-system" } diff --git a/src/domains/paymentoptions-app/04_apim_payment_options.tf b/src/domains/paymentoptions-app/04_apim_payment_options.tf index 3e7f22da2e..813fdf05fe 100644 --- a/src/domains/paymentoptions-app/04_apim_payment_options.tf +++ b/src/domains/paymentoptions-app/04_apim_payment_options.tf @@ -6,7 +6,7 @@ locals { } module "apim_payment_options_product" { - source = "./.terraform/modules/__v3__/api_management_product" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_product?ref=v8.18.0" count = var.is_feature_enabled.paymentoptions ? 1 : 0 product_id = "pagopa_payment_options" diff --git a/src/domains/paymentoptions-app/04_apim_payment_options_mock.tf b/src/domains/paymentoptions-app/04_apim_payment_options_mock.tf index e142259ae6..329281da0f 100644 --- a/src/domains/paymentoptions-app/04_apim_payment_options_mock.tf +++ b/src/domains/paymentoptions-app/04_apim_payment_options_mock.tf @@ -6,7 +6,7 @@ locals { } module "apim_payment_options_mock_product" { - source = "./.terraform/modules/__v3__/api_management_product" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_product?ref=v8.18.0" count = var.is_feature_enabled.paymentoptions_mock ? 1 : 0 product_id = "pagopa-payment-options-mock" @@ -36,7 +36,7 @@ resource "azurerm_api_management_api_version_set" "payment_options_mock_api" { module "apim_api_pay_opt_mock_api" { - source = "./.terraform/modules/__v3__/api_management_api" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_api?ref=v8.18.0" count = var.is_feature_enabled.paymentoptions_mock ? 1 : 0 name = format("%s-pay-opt-mock-api", local.project) diff --git a/src/domains/paymentoptions-app/05_aks_middleware_tools.tf b/src/domains/paymentoptions-app/05_aks_middleware_tools.tf index c391298e4f..83a33add1e 100644 --- a/src/domains/paymentoptions-app/05_aks_middleware_tools.tf +++ b/src/domains/paymentoptions-app/05_aks_middleware_tools.tf @@ -1,39 +1,38 @@ module "tls_checker" { - source = "./.terraform/modules/__v3__/tls_checker" - - https_endpoint = local.domain_hostname - alert_name = local.domain_hostname - alert_enabled = true - helm_chart_present = true - namespace = kubernetes_namespace.namespace.metadata[0].name - location_string = var.location_string - kv_secret_name_for_application_insights_connection_string = "appinsights-instrumentation-key" - application_insights_resource_group = data.azurerm_resource_group.monitor_italy_rg.name - application_insights_id = data.azurerm_application_insights.application_insights_italy.id - application_insights_action_group_ids = [data.azurerm_monitor_action_group.slack.id, data.azurerm_monitor_action_group.email.id] - keyvault_name = data.azurerm_key_vault.kv.name - keyvault_tenant_id = data.azurerm_client_config.current.tenant_id - - workload_identity_enabled = true - workload_identity_service_account_name = module.workload_identity.workload_identity_service_account_name - workload_identity_client_id = module.workload_identity.workload_identity_client_id - - depends_on = [module.workload_identity] + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//tls_checker?ref=v8.22.0" + + https_endpoint = local.domain_hostname + alert_name = local.domain_hostname + alert_enabled = true + helm_chart_present = true + namespace = kubernetes_namespace.namespace.metadata[0].name + location_string = var.location_string + kv_secret_name_for_application_insights_connection_string = "app-insight-connection-string" + application_insights_resource_group = data.azurerm_resource_group.monitor_italy_rg.name + application_insights_id = data.azurerm_application_insights.application_insights_italy.id + application_insights_action_group_ids = [data.azurerm_monitor_action_group.slack.id, data.azurerm_monitor_action_group.email.id] + keyvault_name = data.azurerm_key_vault.kv.name + keyvault_tenant_id = data.azurerm_client_config.current.tenant_id } -module "cert_mounter" { - source = "./.terraform/modules/__v3__/cert_mounter" - - namespace = var.domain - certificate_name = replace(local.domain_hostname, ".", "-"), - kv_name = data.azurerm_key_vault.kv.name - tenant_id = data.azurerm_subscription.current.tenant_id - - workload_identity_enabled = true - workload_identity_service_account_name = module.workload_identity.workload_identity_service_account_name - workload_identity_client_id = module.workload_identity.workload_identity_client_id - - depends_on = [module.workload_identity] +resource "helm_release" "cert_mounter" { + name = "cert-mounter-blueprint" + repository = "https://pagopa.github.io/aks-helm-cert-mounter-blueprint" + chart = "cert-mounter-blueprint" + version = "1.0.4" + namespace = var.domain + timeout = 120 + force_update = true + + values = [ + templatefile("${path.root}/helm/cert-mounter.yaml.tpl", { + NAMESPACE = var.domain, + DOMAIN = var.domain, + CERTIFICATE_NAME = replace(local.domain_hostname, ".", "-"), + ENV_SHORT = var.env_short, + KV_NAME = data.azurerm_key_vault.kv.name + }) + ] } resource "helm_release" "reloader" { diff --git a/src/domains/paymentoptions-app/99_main.tf b/src/domains/paymentoptions-app/99_main.tf index 70cb67af7e..8bf0b91ba8 100644 --- a/src/domains/paymentoptions-app/99_main.tf +++ b/src/domains/paymentoptions-app/99_main.tf @@ -3,23 +3,23 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "<= 3.116.0" + version = "<= 3.106.0" } azuread = { source = "hashicorp/azuread" - version = "<= 3.0.2" + version = "<= 2.47.0" } null = { source = "hashicorp/null" - version = "<= 3.2.3" + version = "<= 3.2.1" } kubernetes = { source = "hashicorp/kubernetes" - version = "<= 2.33.0" + version = "<= 2.29.0" } helm = { source = "hashicorp/helm" - version = "<= 2.16.0" + version = "<= 2.12.1" } } @@ -47,8 +47,3 @@ provider "helm" { config_path = "${var.k8s_kube_config_path_prefix}/config-${local.aks_name}" } } - -module "__v3__" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" -} - diff --git a/src/domains/paymentoptions-common/.terraform.lock.hcl b/src/domains/paymentoptions-common/.terraform.lock.hcl new file mode 100644 index 0000000000..c1bc80ffb2 --- /dev/null +++ b/src/domains/paymentoptions-common/.terraform.lock.hcl @@ -0,0 +1,65 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "2.47.0" + constraints = "<= 2.47.0" + hashes = [ + "h1:g8+gBFM4QVOEQFqAEs5pR6iXpbGvgPvcEi1evHwziyw=", + "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", + "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", + "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", + "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", + "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", + "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", + "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", + "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", + "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", + "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", + "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.106.0" + constraints = "~> 3.30, <= 3.106.0" + hashes = [ + "h1:6t9Nz9tYAR9BfHZ8yc56m+GKRl0nriwjQ5DyA0/TnCs=", + "h1:Mxe1/I27IZK3BP6cm84Gt0+7PXd2EDaDUMxuljm/rUA=", + "zh:07980d6fdc40c0adb670c8413a5c667917d6dbb51fcedc467c35d64c2f3a1f47", + "zh:2e6e8491b1f089644b0d23f8da83398f1e10cf5a62b16efcef2b5454fe923038", + "zh:450dbd72821c5619cc3bcdc20fdd0e29515147e44b733f9c79d3a75851810055", + "zh:5e234c0a2f3c9677ea72b2a6e6ca90defb99fab29ae565f5d1f70728ba4ba78f", + "zh:83fd042ece6977429d79affd03d6ce963d2f122604dbf15a1abf203d7a7bbc8a", + "zh:93027e1f66b3bf83398d572d4e6f6e7777330c78c54da3226dadd50fd868ada9", + "zh:ae3d1dd66140c303df97d93c47a60f16735ce17cf156f45475dcee4a7360af5b", + "zh:daf9d2eb89e785458a76b88bf2ef0696c472094c77cc9cff3b3ea4b885c5a482", + "zh:dd46370141651e6549da6d85e25c7a6770c47581bbaaa27eda2886d41d849747", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f77405c0d8f6e0d93d9da83256b3b02c164bad4c791ed9604310ff02ae086ad1", + "zh:ffa769147bda833aef8802e3a391bd175ec749862764d61cbdaa8200d5b8f893", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.2" + constraints = "<= 3.2.2" + hashes = [ + "h1:IMVAUHKoydFrlPrl9OzasDnw/8ntZFerCC9iXw1rXQY=", + "h1:vWAsYRd7MjYr3adj8BVKRohVfHpWQdvkIwUQ2Jf5FVM=", + "zh:3248aae6a2198f3ec8394218d05bd5e42be59f43a3a7c0b71c66ec0df08b69e7", + "zh:32b1aaa1c3013d33c245493f4a65465eab9436b454d250102729321a44c8ab9a", + "zh:38eff7e470acb48f66380a73a5c7cdd76cc9b9c9ba9a7249c7991488abe22fe3", + "zh:4c2f1faee67af104f5f9e711c4574ff4d298afaa8a420680b0cb55d7bbc65606", + "zh:544b33b757c0b954dbb87db83a5ad921edd61f02f1dc86c6186a5ea86465b546", + "zh:696cf785090e1e8cf1587499516b0494f47413b43cb99877ad97f5d0de3dc539", + "zh:6e301f34757b5d265ae44467d95306d61bef5e41930be1365f5a8dcf80f59452", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:913a929070c819e59e94bb37a2a253c228f83921136ff4a7aa1a178c7cce5422", + "zh:aa9015926cd152425dbf86d1abdbc74bfe0e1ba3d26b3db35051d7b9ca9f72ae", + "zh:bb04798b016e1e1d49bcc76d62c53b56c88c63d6f2dfe38821afef17c416a0e1", + "zh:c23084e1b23577de22603cff752e59128d83cfecc2e6819edadd8cf7a10af11e", + ] +} diff --git a/src/domains/paymentoptions-common/03_eventhub.tf b/src/domains/paymentoptions-common/03_eventhub.tf index e8d3ee1f9e..b2693b1ba8 100644 --- a/src/domains/paymentoptions-common/03_eventhub.tf +++ b/src/domains/paymentoptions-common/03_eventhub.tf @@ -6,7 +6,7 @@ resource "azurerm_resource_group" "eventhub_ita_rg" { } module "eventhub_namespace" { - source = "./.terraform/modules/__v3__/eventhub" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub?ref=v8.22.0" name = "${local.project}-evh" location = var.location resource_group_name = azurerm_resource_group.eventhub_ita_rg.name @@ -52,7 +52,7 @@ module "eventhub_namespace" { # CONFIGURATION # module "eventhub_paymentoptions_configuration" { - source = "./.terraform/modules/__v3__/eventhub_configuration" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration?ref=v8.22.0" count = var.is_feature_enabled.eventhub ? 1 : 0 event_hub_namespace_name = module.eventhub_namespace.name diff --git a/src/domains/paymentoptions-common/10_github_identity.tf b/src/domains/paymentoptions-common/10_github_identity.tf index e77e55ee1a..41e54948b3 100644 --- a/src/domains/paymentoptions-common/10_github_identity.tf +++ b/src/domains/paymentoptions-common/10_github_identity.tf @@ -59,7 +59,7 @@ locals { # create a module for each 20 repos module "identity_cd_01" { - source = "./.terraform/modules/__v3__/github_federated_identity" + source = "github.com/pagopa/terraform-azurerm-v3//github_federated_identity?ref=v8.22.0" # pagopa---github--identity prefix = var.prefix env_short = var.env_short @@ -131,7 +131,7 @@ resource "null_resource" "github_runner_app_permissions_to_namespace_cd_01" { # create a module for each 20 repos module "identity_pr_01" { - source = "./.terraform/modules/__v3__/github_federated_identity" + source = "github.com/pagopa/terraform-azurerm-v3//github_federated_identity?ref=v8.22.0" prefix = var.prefix env_short = var.env_short domain = "${var.domain}-01-pr" @@ -170,7 +170,7 @@ resource "azurerm_key_vault_access_policy" "gha_pr_iac_managed_identities" { # create a module for each 20 repos module "identity_ref_01" { - source = "./.terraform/modules/__v3__/github_federated_identity" + source = "github.com/pagopa/terraform-azurerm-v3//github_federated_identity?ref=v8.36.1" prefix = var.prefix env_short = var.env_short domain = "${var.domain}-01-ref" diff --git a/src/domains/paymentoptions-common/99_main.tf b/src/domains/paymentoptions-common/99_main.tf index 0e2b27945d..cb415d65d1 100644 --- a/src/domains/paymentoptions-common/99_main.tf +++ b/src/domains/paymentoptions-common/99_main.tf @@ -3,15 +3,15 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "<= 3.116.0" + version = "<= 3.106.0" } azuread = { source = "hashicorp/azuread" - version = "<= 3.0.2" + version = "<= 2.47.0" } null = { source = "hashicorp/null" - version = "<= 3.2.3" + version = "<= 3.2.2" } } @@ -29,7 +29,3 @@ provider "azurerm" { data "azurerm_subscription" "current" {} data "azurerm_client_config" "current" {} - -module "__v3__" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" -} diff --git a/src/domains/paymentoptions-secrets/.terraform.lock.hcl b/src/domains/paymentoptions-secrets/.terraform.lock.hcl new file mode 100644 index 0000000000..a389468af5 --- /dev/null +++ b/src/domains/paymentoptions-secrets/.terraform.lock.hcl @@ -0,0 +1,107 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azuread" { + version = "2.47.0" + constraints = "<= 2.47.0" + hashes = [ + "h1:g8+gBFM4QVOEQFqAEs5pR6iXpbGvgPvcEi1evHwziyw=", + "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", + "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", + "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", + "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", + "zh:35227fad617a0509c64ab5759a8b703b10d244877f1aa5416bfbcc100c96996f", + "zh:357f553f0d78d46a96c7b2ed06d25ee0fc60fc5be19812ccb5d969fa47d62e17", + "zh:58faa2940065137e3e87d02eba59ab5cd7137d7a18caf225e660d1788f274569", + "zh:7308eda0339620fa24f47cedd22221fc2c02cab9d5be1710c09a783aea84eb3a", + "zh:863eabf7f908a8263e28d8aa2ad1381affd6bb5c67755216781f674ef214100e", + "zh:8b95b595a7c14ed7b56194d03cdec253527e7a146c1c58961be09e6b5c50baee", + "zh:afbca6b4fac9a0a488bc22ff9e51a8f14e986137d25275068fd932f379a51d57", + "zh:c6aadec4c81a44c3ffc22c2d90ffc6706bf5a9a903a395d896477516f4be6cbb", + "zh:e54a59de7d4ef0f3a18f91fed0b54a2bce18257ae2ee1df8a88226e1023c5811", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "3.106.0" + constraints = "~> 3.30, <= 3.106.0" + hashes = [ + "h1:6t9Nz9tYAR9BfHZ8yc56m+GKRl0nriwjQ5DyA0/TnCs=", + "h1:Mxe1/I27IZK3BP6cm84Gt0+7PXd2EDaDUMxuljm/rUA=", + "zh:07980d6fdc40c0adb670c8413a5c667917d6dbb51fcedc467c35d64c2f3a1f47", + "zh:2e6e8491b1f089644b0d23f8da83398f1e10cf5a62b16efcef2b5454fe923038", + "zh:450dbd72821c5619cc3bcdc20fdd0e29515147e44b733f9c79d3a75851810055", + "zh:5e234c0a2f3c9677ea72b2a6e6ca90defb99fab29ae565f5d1f70728ba4ba78f", + "zh:83fd042ece6977429d79affd03d6ce963d2f122604dbf15a1abf203d7a7bbc8a", + "zh:93027e1f66b3bf83398d572d4e6f6e7777330c78c54da3226dadd50fd868ada9", + "zh:ae3d1dd66140c303df97d93c47a60f16735ce17cf156f45475dcee4a7360af5b", + "zh:daf9d2eb89e785458a76b88bf2ef0696c472094c77cc9cff3b3ea4b885c5a482", + "zh:dd46370141651e6549da6d85e25c7a6770c47581bbaaa27eda2886d41d849747", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f77405c0d8f6e0d93d9da83256b3b02c164bad4c791ed9604310ff02ae086ad1", + "zh:ffa769147bda833aef8802e3a391bd175ec749862764d61cbdaa8200d5b8f893", + ] +} + +provider "registry.terraform.io/hashicorp/external" { + version = "2.2.3" + constraints = "<= 2.2.3" + hashes = [ + "h1:648ZjJR81c2W1OLtYmUQa9/1rGr3vvZSuX9dR1ucGWY=", + "h1:D2RKjqoU26isFINpmeKG9NS0LvkPmrQkNXeYO2TdgyA=", + "zh:184ecd339d764de845db0e5b8a9c87893dcd0c9d822167f73658f89d80ec31c9", + "zh:2661eaca31d17d6bbb18a8f673bbfe3fe1b9b7326e60d0ceb302017003274e3c", + "zh:2c0a180f6d1fc2ba6e03f7dfc5f73b617e45408681f75bca75aa82f3796df0e4", + "zh:4b92ae44c6baef4c4952c47be00541055cb5280dd3bc8031dba5a1b2ee982387", + "zh:5641694d5daf3893d7ea90be03b6fa575211a08814ffe70998d5adb8b59cdc0a", + "zh:5bd55a2be8a1c20d732ac9c604b839e1cadc8c49006315dffa4d709b6874df32", + "zh:6e0ef5d11e1597202424b7d69b9da7b881494c9b13a3d4026fc47012dc651c79", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:9e19f89fa25004d3b926a8d15ea630b4bde62f1fa4ed5e11a3d27aabddb77353", + "zh:b763efdd69fd097616b4a4c89cf333b4cee9699ac6432d73d2756f8335d1213f", + "zh:e3b561efdee510b2b445f76a52a902c52bee8e13095e7f4bed7c80f10f8d294a", + "zh:fe660bb8781ee043a093b9a20e53069974475dcaa5791a1f45fd03c61a26478a", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.16.1" + constraints = "<= 2.16.1" + hashes = [ + "h1:PO4Ye/+lu5hCaUEOtwNOldQYoA0dqL1bcBICIpdlcd8=", + "h1:kO/d+ZMZYM2tNMMFHZqBmVR0MeemoGnI2G2NSN92CrU=", + "zh:06224975f5910d41e73b35a4d5079861da2c24f9353e3ebb015fbb3b3b996b1c", + "zh:2bc400a8d9fe7755cca27c2551564a9e2609cfadc77f526ef855114ee02d446f", + "zh:3a479014187af1d0aec3a1d3d9c09551b801956fe6dd29af1186dec86712731b", + "zh:73fb0a69f1abdb02858b6589f7fab6d989a0f422f7ad95ed662aaa84872d3473", + "zh:a33852cd382cbc8e06d3f6c018b468ad809d24d912d64722e037aed1f9bf39db", + "zh:b533ff2214dca90296b1d22eace7eaa7e3efe5a7ae9da66a112094abc932db4f", + "zh:ddf74d8bb1aeb01dc2c36ef40e2b283d32b2a96db73f6daaf179fa2f10949c80", + "zh:e720f3a15d34e795fa9ff90bc755e838ebb4aef894aa2a423fb16dfa6d6b0667", + "zh:e789ae70a658800cb0a19ef7e4e9b26b5a38a92b43d1f41d64fc8bb46539cefb", + "zh:e8aed7dc0bd8f843d607dee5f72640dbef6835a8b1c6ea12cea5b4ec53e463f7", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:fb3ac4f43c8b0dfc0b0103dd0f062ea72b3a34518d4c8808e3a44c9a3dd5f024", + ] +} + +provider "registry.terraform.io/hashicorp/null" { + version = "3.2.1" + constraints = "~> 3.2, <= 3.2.1" + hashes = [ + "h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=", + "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", + "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", + "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", + "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", + "zh:74cb22c6700e48486b7cabefa10b33b801dfcab56f1a6ac9b6624531f3d36ea3", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:79e553aff77f1cfa9012a2218b8238dd672ea5e1b2924775ac9ac24d2a75c238", + "zh:a1e06ddda0b5ac48f7e7c7d59e1ab5a4073bbcf876c73c0299e4610ed53859dc", + "zh:c37a97090f1a82222925d45d84483b2aa702ef7ab66532af6cbcfb567818b970", + "zh:e4453fbebf90c53ca3323a92e7ca0f9961427d2f0ce0d2b65523cc04d5d999c2", + "zh:e80a746921946d8b6761e77305b752ad188da60688cfd2059322875d363be5f5", + "zh:fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f", + "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", + ] +} diff --git a/src/domains/paymentoptions-secrets/01_keyvault.tf b/src/domains/paymentoptions-secrets/01_keyvault.tf index c91ffe6e9c..28af4e56e2 100644 --- a/src/domains/paymentoptions-secrets/01_keyvault.tf +++ b/src/domains/paymentoptions-secrets/01_keyvault.tf @@ -6,7 +6,7 @@ resource "azurerm_resource_group" "sec_rg" { } module "key_vault" { - source = "./.terraform/modules/__v3__/key_vault" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//key_vault?ref=v8.22.0" name = "${local.product}-${var.location_short}-${var.domain}-kv" location = azurerm_resource_group.sec_rg.location @@ -92,7 +92,7 @@ resource "azurerm_key_vault_access_policy" "azdevops_iac_policy" { # create json letsencrypt inside kv # requierd: Docker module "letsencrypt_paymentoptions" { - source = "./.terraform/modules/__v3__/letsencrypt_credential" + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git///letsencrypt_credential?ref=v8.44.0" prefix = var.prefix env = var.env_short diff --git a/src/domains/paymentoptions-secrets/99_main.tf b/src/domains/paymentoptions-secrets/99_main.tf index ba2156d6dd..93ec8f61ca 100644 --- a/src/domains/paymentoptions-secrets/99_main.tf +++ b/src/domains/paymentoptions-secrets/99_main.tf @@ -2,23 +2,23 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "<= 3.116.0" + version = "<= 3.106.0" } azuread = { source = "hashicorp/azuread" - version = "<= 3.0.2" + version = "<= 2.47.0" } null = { source = "hashicorp/null" - version = "<= 3.2.3" + version = "<= 3.2.1" } external = { source = "hashicorp/external" - version = "<= 2.3.4" + version = "<= 2.2.3" } kubernetes = { source = "hashicorp/kubernetes" - version = "<= 2.33.0" + version = "<= 2.16.1" } } @@ -41,7 +41,3 @@ provider "kubernetes" { data "azurerm_subscription" "current" {} data "azurerm_client_config" "current" {} - -module "__v3__" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=15bbe5eb512bc0fa8f06ed28e0cca754b868743a" -} From 6efa8db37f1770796790dde730ae3605f7d88d2a Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 4 Nov 2024 15:25:32 +0100 Subject: [PATCH 27/55] align config --- .../observability/env/prod/terraform.tfvars | 225 ++++++++++-------- .../observability/env/uat/terraform.tfvars | 221 +++++++++-------- 2 files changed, 240 insertions(+), 206 deletions(-) diff --git a/src/domains/observability/env/prod/terraform.tfvars b/src/domains/observability/env/prod/terraform.tfvars index 04c5611268..d0f6de9cc1 100644 --- a/src/domains/observability/env/prod/terraform.tfvars +++ b/src/domains/observability/env/prod/terraform.tfvars @@ -261,119 +261,136 @@ ehns_metric_alerts = { }, } + + + eventhubs_gpd = [ - { - name = "connect-cluster-offsets" # debezium internal use - partitions = 32 - message_retention = 7 - consumers = ["connect-cluster-offsets"] - keys = [ - { - name = "connect-cluster-offsets" - listen = true - send = true - manage = false - } - ] - }, - { - name = "connect-cluster-status" # debezium internal use - partitions = 32 - message_retention = 7 - consumers = ["connect-cluster-offsets"] - keys = [ - { - name = "connect-cluster-status" - listen = true - send = true - manage = false - } - ] - }, - { - name = "connect-cluster-configs" # debezium internal use - partitions = 32 - message_retention = 7 - consumers = ["connect-cluster-configs"] - keys = [ - { - name = "connect-cluster-configs" - listen = true - send = true - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.payment_option" - partitions = 32 - message_retention = 7 - consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] - keys = [ - { - name = "gpd-ingestion.apd.payment_option-rx-dl" - listen = true - send = false - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.payment_option_metadata" - partitions = 32 - message_retention = 7 - consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] - keys = [ - { - name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" - listen = true - send = false - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.payment_position" - partitions = 32 - message_retention = 7 - consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] + { + name = "test-evh" # test + partitions = 1 + message_retention = 1 + consumers = ["test-evh"] keys = [ { - name = "gpd-ingestion.apd.payment_position-rx-dl" + name = "test-evh" listen = true send = true manage = false } ] }, - { - name = "gpd-ingestion.apd.transfer" - partitions = 32 - message_retention = 7 - consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] - keys = [ - { - name = "gpd-ingestion.apd.transfer-rx-dl" - listen = true - send = false - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.transfer_metadata" - partitions = 32 - message_retention = 7 - consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] - keys = [ - { - name = "gpd-ingestion.apd.transfer_metadata-rx-dl" - listen = true - send = false - manage = false - } - ] - }, + # { + # name = "connect-cluster-offsets" # debezium internal use + # partitions = 32 + # message_retention = 7 + # consumers = ["connect-cluster-offsets"] + # keys = [ + # { + # name = "connect-cluster-offsets" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "connect-cluster-status" # debezium internal use + # partitions = 32 + # message_retention = 7 + # consumers = ["connect-cluster-offsets"] + # keys = [ + # { + # name = "connect-cluster-status" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "connect-cluster-configs" # debezium internal use + # partitions = 32 + # message_retention = 7 + # consumers = ["connect-cluster-configs"] + # keys = [ + # { + # name = "connect-cluster-configs" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.payment_option" + # partitions = 32 + # message_retention = 7 + # consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] + # keys = [ + # { + # name = "gpd-ingestion.apd.payment_option-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.payment_option_metadata" + # partitions = 32 + # message_retention = 7 + # consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.payment_position" + # partitions = 32 + # message_retention = 7 + # consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.payment_position-rx-dl" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.transfer" + # partitions = 32 + # message_retention = 7 + # consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.transfer-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.transfer_metadata" + # partitions = 32 + # message_retention = 7 + # consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.transfer_metadata-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, ] diff --git a/src/domains/observability/env/uat/terraform.tfvars b/src/domains/observability/env/uat/terraform.tfvars index c23da5e0d0..6c17c234ee 100644 --- a/src/domains/observability/env/uat/terraform.tfvars +++ b/src/domains/observability/env/uat/terraform.tfvars @@ -281,119 +281,136 @@ ehns_metric_alerts = { }, } + + + eventhubs_gpd = [ - { - name = "connect-cluster-offsets" # debezium internal use + { + name = "test-evh" # test partitions = 1 message_retention = 1 - consumers = ["connect-cluster-offsets"] + consumers = ["test-evh"] keys = [ { - name = "connect-cluster-offsets" + name = "test-evh" listen = true send = true manage = false } ] }, - { - name = "connect-cluster-status" # debezium internal use - partitions = 1 - message_retention = 1 - consumers = ["connect-cluster-offsets"] - keys = [ - { - name = "connect-cluster-status" - listen = true - send = true - manage = false - } - ] - }, - { - name = "connect-cluster-configs" # debezium internal use - partitions = 1 - message_retention = 1 - consumers = ["connect-cluster-configs"] - keys = [ - { - name = "connect-cluster-configs" - listen = true - send = true - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.payment_option" - partitions = 1 - message_retention = 1 - consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] - keys = [ - { - name = "gpd-ingestion.apd.payment_option-rx-dl" - listen = true - send = false - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.payment_option_metadata" - partitions = 1 - message_retention = 1 - consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] - keys = [ - { - name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" - listen = true - send = false - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.payment_position" - partitions = 1 - message_retention = 1 - consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] - keys = [ - { - name = "gpd-ingestion.apd.payment_position-rx-dl" - listen = true - send = true - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.transfer" - partitions = 1 - message_retention = 1 - consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] - keys = [ - { - name = "gpd-ingestion.apd.transfer-rx-dl" - listen = true - send = false - manage = false - } - ] - }, - { - name = "gpd-ingestion.apd.transfer_metadata" - partitions = 1 - message_retention = 1 - consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] - keys = [ - { - name = "gpd-ingestion.apd.transfer_metadata-rx-dl" - listen = true - send = false - manage = false - } - ] - }, + # { + # name = "connect-cluster-offsets" # debezium internal use + # partitions = 1 + # message_retention = 1 + # consumers = ["connect-cluster-offsets"] + # keys = [ + # { + # name = "connect-cluster-offsets" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "connect-cluster-status" # debezium internal use + # partitions = 1 + # message_retention = 1 + # consumers = ["connect-cluster-offsets"] + # keys = [ + # { + # name = "connect-cluster-status" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "connect-cluster-configs" # debezium internal use + # partitions = 1 + # message_retention = 1 + # consumers = ["connect-cluster-configs"] + # keys = [ + # { + # name = "connect-cluster-configs" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.payment_option" + # partitions = 1 + # message_retention = 1 + # consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] + # keys = [ + # { + # name = "gpd-ingestion.apd.payment_option-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.payment_option_metadata" + # partitions = 1 + # message_retention = 1 + # consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.payment_position" + # partitions = 1 + # message_retention = 1 + # consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.payment_position-rx-dl" + # listen = true + # send = true + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.transfer" + # partitions = 1 + # message_retention = 1 + # consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.transfer-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, + # { + # name = "gpd-ingestion.apd.transfer_metadata" + # partitions = 1 + # message_retention = 1 + # consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] + # keys = [ + # { + # name = "gpd-ingestion.apd.transfer_metadata-rx-dl" + # listen = true + # send = false + # manage = false + # } + # ] + # }, ] From 64e2c0c504af74be33fd75297560b26ccbd13c1f Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 4 Nov 2024 15:33:15 +0100 Subject: [PATCH 28/55] change evh to public --- src/domains/observability/env/prod/terraform.tfvars | 2 +- src/domains/observability/env/uat/terraform.tfvars | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/domains/observability/env/prod/terraform.tfvars b/src/domains/observability/env/prod/terraform.tfvars index d0f6de9cc1..5bfe4f1a3d 100644 --- a/src/domains/observability/env/prod/terraform.tfvars +++ b/src/domains/observability/env/prod/terraform.tfvars @@ -74,7 +74,7 @@ ehns_capacity = 5 ehns_alerts_enabled = true ehns_zone_redundant = true -ehns_public_network_access = false +ehns_public_network_access = true ehns_private_endpoint_is_present = true eventhubs = [ diff --git a/src/domains/observability/env/uat/terraform.tfvars b/src/domains/observability/env/uat/terraform.tfvars index 6c17c234ee..9a043030d5 100644 --- a/src/domains/observability/env/uat/terraform.tfvars +++ b/src/domains/observability/env/uat/terraform.tfvars @@ -74,7 +74,7 @@ ehns_capacity = 1 ehns_alerts_enabled = false ehns_zone_redundant = false -ehns_public_network_access = false +ehns_public_network_access = true ehns_private_endpoint_is_present = true eventhubs = [ From 44271bc48d3c63a4337ec35ceca6ef3b80525356 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Tue, 5 Nov 2024 00:14:51 +0100 Subject: [PATCH 29/55] fix --- src/domains/gps-app/00_data.tf | 10 +-- src/domains/gps-app/05_debezium_connect.tf | 38 ++++++------ src/domains/gps-app/99_variables.tf | 62 +++++++++---------- src/domains/gps-app/README.md | 42 ++++++++++--- src/domains/gps-app/yaml/kafka-connect.yaml | 4 +- .../gps-app/yaml/postgres-connector.yaml | 1 + 6 files changed, 94 insertions(+), 63 deletions(-) diff --git a/src/domains/gps-app/00_data.tf b/src/domains/gps-app/00_data.tf index b92843b3f2..a963c9d253 100644 --- a/src/domains/gps-app/00_data.tf +++ b/src/domains/gps-app/00_data.tf @@ -22,11 +22,11 @@ data "azurerm_key_vault_secret" "gpd_db_pwd" { key_vault_id = data.azurerm_key_vault.gps_kv.id } -data "azurerm_postgresql_server" "postgresql" { - count = var.env_short == "d" ? 1 : 0 - name = format("%s-gpd-postgresql", local.product) - resource_group_name = format("%s-gpd-rg", local.product) -} +# data "azurerm_postgresql_server" "postgresql" { +# count = var.env_short == "d" ? 1 : 0 +# name = format("%s-gpd-postgresql", local.product) +# resource_group_name = format("%s-gpd-rg", local.product) +# } data "azurerm_postgresql_flexible_server" "postgres_flexible_server_private" { count = var.env_short != "d" ? 1 : 0 diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index 7278442ef1..1905ee476f 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -1,10 +1,12 @@ # https://debezium.io/documentation/reference/stable/operations/kubernetes.html#_creating_a_debezium_connector data "azurerm_key_vault_secret" "pgres_gpd_cdc_login" { + # name = "db-apd-user-name" name = "cdc-logical-replication-apd-user" key_vault_id = data.azurerm_key_vault.kv.id } data "azurerm_key_vault_secret" "pgres_gpd_cdc_pwd" { + # name = "db-apd-user-password" name = "cdc-logical-replication-apd-pwd" key_vault_id = data.azurerm_key_vault.kv.id } @@ -21,7 +23,7 @@ resource "helm_release" "strimzi-kafka-operator" { chart = "strimzi-kafka-operator" repository = "oci://quay.io/strimzi-helm" version = "0.43.0" - namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name } locals { @@ -34,12 +36,12 @@ locals { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name }) - debezium_secrets_yaml = templatefile("${path.module}/yaml/debezium-secrets.yaml", { - namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - username = base64encode(data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value) - password = base64encode(data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value) - connection_string = base64encode(data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string) - }) + debezium_secrets_yaml = templatefile("${path.module}/yaml/debezium-secrets.yaml", { + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + username = base64encode(data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value) + password = base64encode(data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value) + connection_string = base64encode(data.azurerm_eventhub_namespace_authorization_rule.cdc_connection_string.primary_connection_string) + }) # zookeeper_yaml = templatefile("${path.module}/yaml/zookeeper.yaml", { # namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name @@ -57,20 +59,20 @@ locals { # https://learn.microsoft.com/it-it/azure/event-hubs/event-hubs-kafka-connect-debezium#configure-kafka-connect-for-event-hubs kafka_connect_yaml = templatefile("${path.module}/yaml/kafka-connect.yaml", { - namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - replicas = var.replicas - request_memory = var.request_memory - request_cpu = var.request_cpu - limits_memory = var.limits_memory - limits_cpu = var.limits_cpu - bootstrap_servers = "pagopa-${var.env_short}-itn-observ-gpd-evh.servicebus.windows.net:9093" - container_registry = var.container_registry + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + replicas = var.replicas + request_memory = var.request_memory + request_cpu = var.request_cpu + limits_memory = var.limits_memory + limits_cpu = var.limits_cpu + bootstrap_servers = "pagopa-${var.env_short}-itn-observ-gpd-evh.servicebus.windows.net:9093" + container_registry = var.container_registry }) postgres_connector_yaml = templatefile("${path.module}/yaml/postgres-connector.yaml", { - namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - postgres_hostname = "pagopa-${var.env_short}-gpd-pgflex.postgres.database.azure.com" - + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + postgres_hostname = "pagopa-${var.env_short}-gpd-pgflex.postgres.database.azure.com" + postgres_port = 5432 postgres_db_name = var.postgres_db_name postgres_topic_prefix = "azcligpd" diff --git a/src/domains/gps-app/99_variables.tf b/src/domains/gps-app/99_variables.tf index 2b653978de..cb34fc5c2d 100644 --- a/src/domains/gps-app/99_variables.tf +++ b/src/domains/gps-app/99_variables.tf @@ -446,96 +446,96 @@ variable "flag_responsetime_alert" { ### debezium kafka conn variable "zookeeper_replicas" { - type = number + type = number description = "Zookeeper Replicas" - default = 1 + default = 1 } variable "zookeeper_request_memory" { - type = string + type = string description = "Zookeeper Request Memory" - default = "512m" + default = "512m" } variable "zookeeper_request_cpu" { - type = string + type = string description = "Zookeeper Request CPU" - default = "0.5" + default = "0.5" } variable "zookeeper_limits_memory" { - type = string + type = string description = "Zookeeper Limit Memory" - default = "512mi" + default = "512mi" } variable "zookeeper_limits_cpu" { - type = string + type = string description = "Zookeeper Limit CPU" - default = "0.5" + default = "0.5" } variable "zookeeper_jvm_xms" { - type = string + type = string description = "Zookeeper Jvm Xms" - default = "512mi" + default = "512mi" } variable "zookeeper_jvm_xmx" { - type = string + type = string description = "Zookeeper Jvm Xmx" - default = "512mi" + default = "512mi" } variable "zookeeper_storage_size" { - type = string + type = string description = "Zookeeper Storage Size" - default = "100Gi" + default = "100Gi" } variable "container_registry" { - type = string + type = string description = "Container Registry" } variable "postgres_db_name" { - type = string + type = string description = "Postgres Database Name" - default = "apd" + default = "apd" } variable "tasks_max" { - type = string + type = string description = "Number of tasks" - default = "1" + default = "1" } variable "replicas" { - type = number + type = number description = "Number of replicas in cluster" - default = 1 + default = 1 } variable "request_memory" { - type = string + type = string description = "Connect Request Memory" - default = "512m" + default = "512m" } variable "request_cpu" { - type = string + type = string description = "Connect Request CPU" - default = "0.5" + default = "0.5" } variable "limits_memory" { - type = string + type = string description = "Connect Limit Memory" - default = "512mi" + default = "512mi" } variable "limits_cpu" { - type = string + type = string description = "Connect Limit CPU" - default = "0.5" + default = "0.5" } diff --git a/src/domains/gps-app/README.md b/src/domains/gps-app/README.md index aa69eda115..8e1f4ef4fb 100644 --- a/src/domains/gps-app/README.md +++ b/src/domains/gps-app/README.md @@ -10,6 +10,7 @@ | [azuread](#requirement\_azuread) | <= 2.21.0 | | [azurerm](#requirement\_azurerm) | <= 3.45.0 | | [helm](#requirement\_helm) | <= 2.5.1 | +| [kubectl](#requirement\_kubectl) | 1.14.0 | | [kubernetes](#requirement\_kubernetes) | <= 2.11.0 | | [null](#requirement\_null) | <= 3.2.1 | @@ -17,6 +18,7 @@ | Name | Source | Version | |------|--------|---------| +| [\_\_v3\_\_](#module\_\_\_v3\_\_) | git::https://github.com/pagopa/terraform-azurerm-v3 | 15bbe5eb512bc0fa8f06ed28e0cca754b868743a | | [apim\_aca\_integration\_product](#module\_apim\_aca\_integration\_product) | git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_product | v6.4.1 | | [apim\_api\_debt\_positions\_api\_v1](#module\_apim\_api\_debt\_positions\_api\_v1) | git::https://github.com/pagopa/terraform-azurerm-v3//api_management_api | v6.11.2 | | [apim\_api\_debt\_positions\_api\_v2](#module\_apim\_api\_debt\_positions\_api\_v2) | git::https://github.com/pagopa/terraform-azurerm-v3//api_management_api | v6.11.2 | @@ -104,12 +106,20 @@ | [azurerm_resource_group.gpd_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [helm_release.cert_mounter](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.reloader](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.strimzi-kafka-operator](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubectl_manifest.debezium_role](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource | +| [kubectl_manifest.debezium_secrets](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource | +| [kubectl_manifest.debezoum_rbac](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource | +| [kubectl_manifest.kafka_connect](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource | +| [kubectl_manifest.postgres_connector](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource | | [kubernetes_namespace.namespace](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | | [kubernetes_namespace.namespace_system](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | | [kubernetes_pod_disruption_budget_v1.gps](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/pod_disruption_budget_v1) | resource | | [kubernetes_role_binding.deployer_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding) | resource | | [kubernetes_role_binding.system_deployer_binding](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/role_binding) | resource | | [kubernetes_service_account.azure_devops](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account) | resource | +| [null_resource.wait_kafka_connect](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.wait_postgres_connector](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [azuread_group.adgroup_admin](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | | [azuread_group.adgroup_developers](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | | [azuread_group.adgroup_externals](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | @@ -118,6 +128,7 @@ | [azurerm_application_insights.application_insights](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/application_insights) | data source | | [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | | [azurerm_container_registry.acr](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/container_registry) | data source | +| [azurerm_eventhub_namespace_authorization_rule.cdc_connection_string](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_namespace_authorization_rule) | data source | | [azurerm_key_vault.gps_kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault) | data source | | [azurerm_key_vault.kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault) | data source | | [azurerm_key_vault_secret.config_cache_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | @@ -127,13 +138,14 @@ | [azurerm_key_vault_secret.gpd_paa_pwd](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | | [azurerm_key_vault_secret.monitor_notification_email](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | | [azurerm_key_vault_secret.monitor_notification_slack_email](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | +| [azurerm_key_vault_secret.pgres_gpd_cdc_login](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | +| [azurerm_key_vault_secret.pgres_gpd_cdc_pwd](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | | [azurerm_kubernetes_cluster.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_cluster) | data source | | [azurerm_log_analytics_workspace.log_analytics](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/log_analytics_workspace) | data source | | [azurerm_monitor_action_group.email](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | | [azurerm_monitor_action_group.opsgenie](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | | [azurerm_monitor_action_group.slack](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | | [azurerm_postgresql_flexible_server.postgres_flexible_server_private](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/postgresql_flexible_server) | data source | -| [azurerm_postgresql_server.postgresql](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/postgresql_server) | data source | | [azurerm_resource_group.monitor_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | | [azurerm_resource_group.rg_api](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | | [azurerm_subnet.apim_snet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | @@ -151,8 +163,9 @@ | [cidr\_subnet\_gpd](#input\_cidr\_subnet\_gpd) | Address prefixes subnet gpd service | `list(string)` | `null` | no | | [cidr\_subnet\_reporting\_functions](#input\_cidr\_subnet\_reporting\_functions) | Address prefixes subnet reporting\_common function | `list(string)` | `null` | no | | [cname\_record\_name](#input\_cname\_record\_name) | n/a | `string` | n/a | yes | +| [container\_registry](#input\_container\_registry) | Container Registry | `string` | n/a | yes | | [create\_wisp\_converter](#input\_create\_wisp\_converter) | CREATE WISP dismantling system infra | `bool` | `false` | no | -| [ddos\_protection\_plan](#input\_ddos\_protection\_plan) | Network | 
object({
    id     = string
    enable = bool
  })
| `null` | no | +| [ddos\_protection\_plan](#input\_ddos\_protection\_plan) | Network | 
object({
    id     = string
    enable = bool
  })
| `null` | no | | [dns\_zone\_internal\_prefix](#input\_dns\_zone\_internal\_prefix) | The dns subdomain. | `string` | `null` | no | | [domain](#input\_domain) | n/a | `string` | n/a | yes | | [enable\_client\_retry](#input\_enable\_client\_retry) | Enable client retry | `bool` | `false` | no | @@ -160,7 +173,7 @@ | [env\_short](#input\_env\_short) | n/a | `string` | n/a | yes | | [external\_domain](#input\_external\_domain) | Domain for delegation | `string` | `null` | no | | [flag\_responsetime\_alert](#input\_flag\_responsetime\_alert) | Flag to enable if payments-pull response time alert is available | `number` | `0` | no | -| [fn\_app\_storage\_account\_info](#input\_fn\_app\_storage\_account\_info) | n/a | 
object({
    account_kind                      = optional(string, "StorageV2")
    account_tier                      = optional(string, "Standard")
    account_replication_type          = optional(string, "LRS")
    advanced_threat_protection_enable = optional(bool, true)
    access_tier                       = optional(string, "Hot")
  })
| 
{
  "access_tier": "Hot",
  "account_kind": "StorageV2",
  "account_replication_type": "LRS",
  "account_tier": "Standard",
  "advanced_threat_protection_enable": true
}
| no | +| [fn\_app\_storage\_account\_info](#input\_fn\_app\_storage\_account\_info) | n/a | 
object({
    account_kind                      = optional(string, "StorageV2")
    account_tier                      = optional(string, "Standard")
    account_replication_type          = optional(string, "LRS")
    advanced_threat_protection_enable = optional(bool, true)
    access_tier                       = optional(string, "Hot")
  })
| 
{
  "access_tier": "Hot",
  "account_kind": "StorageV2",
  "account_replication_type": "LRS",
  "account_tier": "Standard",
  "advanced_threat_protection_enable": true
}
| no | | [gpd\_always\_on](#input\_gpd\_always\_on) | Always on property | `bool` | `true` | no | | [gpd\_autoscale\_default](#input\_gpd\_autoscale\_default) | The number of instances that are available for scaling if metrics are not available for evaluation. | `number` | `1` | no | | [gpd\_autoscale\_maximum](#input\_gpd\_autoscale\_maximum) | The maximum number of instances for this resource. | `number` | `3` | no | @@ -183,6 +196,8 @@ | [initial\_interval\_millis](#input\_initial\_interval\_millis) | The initial interval in milliseconds | `number` | `500` | no | | [instance](#input\_instance) | One of beta, prod01, prod02 | `string` | n/a | yes | | [k8s\_kube\_config\_path\_prefix](#input\_k8s\_kube\_config\_path\_prefix) | n/a | `string` | `"~/.kube"` | no | +| [limits\_cpu](#input\_limits\_cpu) | Connect Limit CPU | `string` | `"0.5"` | no | +| [limits\_memory](#input\_limits\_memory) | Connect Limit Memory | `string` | `"512mi"` | no | | [location](#input\_location) | One of westeurope, northeurope | `string` | n/a | yes | | [location\_short](#input\_location\_short) | One of wue, neu | `string` | n/a | yes | | [location\_string](#input\_location\_string) | One of West Europe, North Europe | `string` | n/a | yes | @@ -193,9 +208,11 @@ | [monitor\_resource\_group\_name](#input\_monitor\_resource\_group\_name) | Monitor resource group name | `string` | `"pagopa-p-monitor-rg"` | no | | [multiplier](#input\_multiplier) | Multiplier for the client backoff procedure | `number` | `1.5` | no | | [pgbouncer\_enabled](#input\_pgbouncer\_enabled) | Built-in connection pooling solution | `bool` | `false` | no | -| [pod\_disruption\_budgets](#input\_pod\_disruption\_budgets) | Pod disruption budget for domain namespace | 
map(object({
    name         = optional(string, null)
    minAvailable = optional(number, null)
    matchLabels  = optional(map(any), {})
  }))
| `{}` | no | +| [pod\_disruption\_budgets](#input\_pod\_disruption\_budgets) | Pod disruption budget for domain namespace | 
map(object({
    name         = optional(string, null)
    minAvailable = optional(number, null)
    matchLabels  = optional(map(any), {})
  }))
| `{}` | no | +| [postgres\_db\_name](#input\_postgres\_db\_name) | Postgres Database Name | `string` | `"apd"` | no | | [prefix](#input\_prefix) | n/a | `string` | n/a | yes | | [randomization\_factor](#input\_randomization\_factor) | Randomization factor for the backoff procedure | `number` | `0.5` | no | +| [replicas](#input\_replicas) | Number of replicas in cluster | `number` | `1` | no | | [reporting\_analysis\_dotnet\_version](#input\_reporting\_analysis\_dotnet\_version) | n/a | `string` | `null` | no | | [reporting\_analysis\_function\_always\_on](#input\_reporting\_analysis\_function\_always\_on) | Always on property | `bool` | `false` | no | | [reporting\_analysis\_function\_client\_certificate\_mode](#input\_reporting\_analysis\_function\_client\_certificate\_mode) | client\_certificate\_mode property | `string` | `"Required"` | no | @@ -207,12 +224,23 @@ | [reporting\_function\_autoscale\_default](#input\_reporting\_function\_autoscale\_default) | The number of instances that are available for scaling if metrics are not available for evaluation. | `number` | `5` | no | | [reporting\_function\_autoscale\_maximum](#input\_reporting\_function\_autoscale\_maximum) | The maximum number of instances for this resource. | `number` | `10` | no | | [reporting\_function\_autoscale\_minimum](#input\_reporting\_function\_autoscale\_minimum) | The minimum number of instances for this resource. | `number` | `1` | no | -| [reporting\_functions\_app\_sku](#input\_reporting\_functions\_app\_sku) | Reporting functions app plan SKU | 
object({
    kind     = string
    sku_tier = string
    sku_size = string
  })
| n/a | yes | +| [reporting\_functions\_app\_sku](#input\_reporting\_functions\_app\_sku) | Reporting functions app plan SKU | 
object({
    kind     = string
    sku_tier = string
    sku_size = string
  })
| n/a | yes | | [reporting\_service\_dotnet\_version](#input\_reporting\_service\_dotnet\_version) | n/a | `string` | `null` | no | | [reporting\_service\_function\_always\_on](#input\_reporting\_service\_function\_always\_on) | Always on property | `bool` | `false` | no | | [reporting\_service\_image](#input\_reporting\_service\_image) | reporting\_service\_function docker image | `string` | `""` | no | -| [tags](#input\_tags) | n/a | `map(any)` | 
{
  "CreatedBy": "Terraform"
}
| no | -| [tls\_cert\_check\_helm](#input\_tls\_cert\_check\_helm) | tls cert helm chart configuration | 
object({
    chart_version = string,
    image_name    = string,
    image_tag     = string
  })
| n/a | yes | +| [request\_cpu](#input\_request\_cpu) | Connect Request CPU | `string` | `"0.5"` | no | +| [request\_memory](#input\_request\_memory) | Connect Request Memory | `string` | `"512m"` | no | +| [tags](#input\_tags) | n/a | `map(any)` | 
{
  "CreatedBy": "Terraform"
}
| no | +| [tasks\_max](#input\_tasks\_max) | Number of tasks | `string` | `"1"` | no | +| [tls\_cert\_check\_helm](#input\_tls\_cert\_check\_helm) | tls cert helm chart configuration | 
object({
    chart_version = string,
    image_name    = string,
    image_tag     = string
  })
| n/a | yes | +| [zookeeper\_jvm\_xms](#input\_zookeeper\_jvm\_xms) | Zookeeper Jvm Xms | `string` | `"512mi"` | no | +| [zookeeper\_jvm\_xmx](#input\_zookeeper\_jvm\_xmx) | Zookeeper Jvm Xmx | `string` | `"512mi"` | no | +| [zookeeper\_limits\_cpu](#input\_zookeeper\_limits\_cpu) | Zookeeper Limit CPU | `string` | `"0.5"` | no | +| [zookeeper\_limits\_memory](#input\_zookeeper\_limits\_memory) | Zookeeper Limit Memory | `string` | `"512mi"` | no | +| [zookeeper\_replicas](#input\_zookeeper\_replicas) | Zookeeper Replicas | `number` | `1` | no | +| [zookeeper\_request\_cpu](#input\_zookeeper\_request\_cpu) | Zookeeper Request CPU | `string` | `"0.5"` | no | +| [zookeeper\_request\_memory](#input\_zookeeper\_request\_memory) | Zookeeper Request Memory | `string` | `"512m"` | no | +| [zookeeper\_storage\_size](#input\_zookeeper\_storage\_size) | Zookeeper Storage Size | `string` | `"100Gi"` | no | ## Outputs diff --git a/src/domains/gps-app/yaml/kafka-connect.yaml b/src/domains/gps-app/yaml/kafka-connect.yaml index 19ebd11041..c266bd9527 100644 --- a/src/domains/gps-app/yaml/kafka-connect.yaml +++ b/src/domains/gps-app/yaml/kafka-connect.yaml @@ -37,8 +37,8 @@ spec: status.storage.replication.factor: 1 rest.advertised.host.name: connect offset.flush.interval.ms: 10000 - topic.creation.enable: "false" - auto.create.topics.enable: "false" # https://debezium.io/documentation/reference/3.0/configuration/topic-auto-create-config.html#disabling-automatic-topic-creation-for-the-kafka-broker + topic.creation.enable: "true" + auto.create.topics.enable: "true" # https://debezium.io/documentation/reference/3.0/configuration/topic-auto-create-config.html#disabling-automatic-topic-creation-for-the-kafka-broker key.converter: org.apache.kafka.connect.json.JsonConverter value.converter: org.apache.kafka.connect.json.JsonConverter internal.key.converter: org.apache.kafka.connect.json.JsonConverter diff --git a/src/domains/gps-app/yaml/postgres-connector.yaml b/src/domains/gps-app/yaml/postgres-connector.yaml index 57d4914500..d87528b794 100644 --- a/src/domains/gps-app/yaml/postgres-connector.yaml +++ b/src/domains/gps-app/yaml/postgres-connector.yaml @@ -25,3 +25,4 @@ spec: table.include.list: "apd.payment_option,apd.payment_option_metadata,apd.payment_position,apd.transfer,apd.transfer_metadata" plugin.name: "pgoutput" publication.autocreate.mode: "disabled" # shall be create before + # publication.autocreate.mode: "filtered" From c3dd08e27dfb7bf2a43d937a5b222a194ead2f51 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Tue, 5 Nov 2024 00:18:25 +0100 Subject: [PATCH 30/55] add mng to cdc-gpd-connection-string --- .../observability/03_eventhub_msg_gdp.tf | 2 +- .../observability/04_datafactory_dataflow.tf | 840 +++++++++--------- .../observability/04_datafactory_dataset.tf | 32 +- src/domains/observability/99_main.tf | 2 +- src/domains/observability/README.md | 76 +- 5 files changed, 510 insertions(+), 442 deletions(-) diff --git a/src/domains/observability/03_eventhub_msg_gdp.tf b/src/domains/observability/03_eventhub_msg_gdp.tf index 955cfb4161..893b86d984 100644 --- a/src/domains/observability/03_eventhub_msg_gdp.tf +++ b/src/domains/observability/03_eventhub_msg_gdp.tf @@ -60,7 +60,7 @@ resource "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" resource_group_name = azurerm_resource_group.eventhub_observability_rg.name listen = true send = true - manage = false + manage = true } # MS doc configure-cleanup-policy https://learn.microsoft.com/en-us/azure/event-hubs/configure-event-hub-properties#configure-cleanup-policy diff --git a/src/domains/observability/04_datafactory_dataflow.tf b/src/domains/observability/04_datafactory_dataflow.tf index 418c2d9b89..da2f91b523 100644 --- a/src/domains/observability/04_datafactory_dataflow.tf +++ b/src/domains/observability/04_datafactory_dataflow.tf @@ -13,137 +13,137 @@ resource "azapi_resource" "pdnd_cdc_gec_bundles_dataflow" { name = "PDND_CDC_GEC_DATAFLOW" } type = "MappingDataFlow" - typeProperties = { - #script = "" - scriptLines = [ - "source(output(", - " {_rid} as string,", - " {_ts} as long,", - " id as string,", - " idPsp as string,", - " idChannel as string,", - " idBrokerPsp as string,", - " cart as boolean,", - " idCdi as string,", - " abi as string,", - " pspBusinessName as string,", - " urlPolicyPsp as integer,", - " digitalStamp as boolean,", - " digitalStampRestriction as boolean,", - " name as string,", - " description as string,", - " paymentAmount as long,", - " minPaymentAmount as long,", - " maxPaymentAmount as long,", - " paymentType as string,", - " touchpoint as string,", - " type as string,", - " transferCategoryList as string[],", - " validityDateFrom as long[],", - " validityDateTo as long[],", - " insertedDate as long[],", - " lastUpdatedDate as long[],", - " {_etag} as string", - " ),", - " allowSchemaDrift: true,", - " validateSchema: false,", - " inferDriftedColumnTypes: true,", - " container: 'bundles',", - " storeType: 'olap',", - " format: 'document',", - " enableChangeFeed: true,", - " changeFeedStartFromTheBeginning: true,", - " systemColumns: true,", - " captureIntermediateUpdate: false,", - " captureUserDeletes: true,", - " captureTxnTTLDeletes: true,", - " store: 'cosmosDB') ~> cosmosbundles", - "cosmosbundles derive(lastUpdatedDateString = iif(isNull(lastUpdatedDate), '', toString(lastUpdatedDate[1]) + '-' + lpad(toString(lastUpdatedDate[2]), 2, '0') + '-' + lpad(toString(lastUpdatedDate[3]), 2, '0') + 'T' + lpad(toString(lastUpdatedDate[4]), 2, '0') + ':' + lpad(toString(lastUpdatedDate[5]), 2, '0') + ':' + lpad(toString(lastUpdatedDate[6]), 2, '0') + '.' + lpad(toString(lastUpdatedDate[7]), 2, '0')),", - " insertedDateString = iif(isNull(insertedDate), '', toString(insertedDate[1]) + '-' + lpad(toString(insertedDate[2]), 2, '0') + '-' + lpad(toString(insertedDate[3]), 2, '0') + 'T' + lpad(toString(insertedDate[4]), 2, '0') + ':' + lpad(toString(insertedDate[5]), 2, '0') + ':' + lpad(toString(insertedDate[6]), 2, '0') + '.' + lpad(toString(insertedDate[7]), 2, '0')),", - " validityDateFromString = iif(isNull(validityDateFrom), '', toString(validityDateFrom[1]) + '-' + lpad(toString(validityDateFrom[1]), 2, '0') + '-' + lpad(toString(validityDateFrom[1]), 2, '0')),", - " validityDateToString = iif(isNull(validityDateTo), '', toString(validityDateTo[1]) + '-' + lpad(toString(validityDateTo[1]), 2, '0') + '-' + lpad(toString(validityDateTo[1]), 2, '0')),", - " {_etag} = regexReplace({_etag}, '\\\"', '')) ~> transformDateFormat", - "transformDateFormat select(mapColumn(", - " {_rid},", - " {_ts},", - " id,", - " idPsp,", - " idChannel,", - " idBrokerPsp,", - " cart,", - " idCdi,", - " abi,", - " pspBusinessName,", - " urlPolicyPsp,", - " digitalStamp,", - " digitalStampRestriction,", - " name,", - " description,", - " paymentAmount,", - " minPaymentAmount,", - " maxPaymentAmount,", - " paymentType,", - " touchpoint,", - " type,", - " transferCategoryList,", - " {_etag},", - " lastUpdatedDate = lastUpdatedDateString,", - " insertedDate = insertedDateString,", - " validityDateFrom = validityDateFromString,", - " validityDateTo = validityDateToString", - " ),", - " skipDuplicateMapInputs: true,", - " skipDuplicateMapOutputs: true) ~> selectFields", - "selectFields sink(allowSchemaDrift: true,", - " validateSchema: false,", - " format: 'json',", - " container: 'pagopa-${var.env_short}-itn-observ-az-blob-observability-container',", - " folderPath: 'bundles',", - " skipDuplicateMapInputs: true,", - " skipDuplicateMapOutputs: true) ~> afmgecstorage" - ] - sinks = [ - { - description = "Write data to blob storage in json format" - linkedService = { - parameters = {} - referenceName = "afm-gec-${var.env_short}-${var.location_short}-sa-linkedservice" - type = "LinkedServiceReference" - } - name = "afmgecstorage" + typeProperties = { + #script = "" + scriptLines = [ + "source(output(", + " {_rid} as string,", + " {_ts} as long,", + " id as string,", + " idPsp as string,", + " idChannel as string,", + " idBrokerPsp as string,", + " cart as boolean,", + " idCdi as string,", + " abi as string,", + " pspBusinessName as string,", + " urlPolicyPsp as integer,", + " digitalStamp as boolean,", + " digitalStampRestriction as boolean,", + " name as string,", + " description as string,", + " paymentAmount as long,", + " minPaymentAmount as long,", + " maxPaymentAmount as long,", + " paymentType as string,", + " touchpoint as string,", + " type as string,", + " transferCategoryList as string[],", + " validityDateFrom as long[],", + " validityDateTo as long[],", + " insertedDate as long[],", + " lastUpdatedDate as long[],", + " {_etag} as string", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " inferDriftedColumnTypes: true,", + " container: 'bundles',", + " storeType: 'olap',", + " format: 'document',", + " enableChangeFeed: true,", + " changeFeedStartFromTheBeginning: true,", + " systemColumns: true,", + " captureIntermediateUpdate: false,", + " captureUserDeletes: true,", + " captureTxnTTLDeletes: true,", + " store: 'cosmosDB') ~> cosmosbundles", + "cosmosbundles derive(lastUpdatedDateString = iif(isNull(lastUpdatedDate), '', toString(lastUpdatedDate[1]) + '-' + lpad(toString(lastUpdatedDate[2]), 2, '0') + '-' + lpad(toString(lastUpdatedDate[3]), 2, '0') + 'T' + lpad(toString(lastUpdatedDate[4]), 2, '0') + ':' + lpad(toString(lastUpdatedDate[5]), 2, '0') + ':' + lpad(toString(lastUpdatedDate[6]), 2, '0') + '.' + lpad(toString(lastUpdatedDate[7]), 2, '0')),", + " insertedDateString = iif(isNull(insertedDate), '', toString(insertedDate[1]) + '-' + lpad(toString(insertedDate[2]), 2, '0') + '-' + lpad(toString(insertedDate[3]), 2, '0') + 'T' + lpad(toString(insertedDate[4]), 2, '0') + ':' + lpad(toString(insertedDate[5]), 2, '0') + ':' + lpad(toString(insertedDate[6]), 2, '0') + '.' + lpad(toString(insertedDate[7]), 2, '0')),", + " validityDateFromString = iif(isNull(validityDateFrom), '', toString(validityDateFrom[1]) + '-' + lpad(toString(validityDateFrom[1]), 2, '0') + '-' + lpad(toString(validityDateFrom[1]), 2, '0')),", + " validityDateToString = iif(isNull(validityDateTo), '', toString(validityDateTo[1]) + '-' + lpad(toString(validityDateTo[1]), 2, '0') + '-' + lpad(toString(validityDateTo[1]), 2, '0')),", + " {_etag} = regexReplace({_etag}, '\\\"', '')) ~> transformDateFormat", + "transformDateFormat select(mapColumn(", + " {_rid},", + " {_ts},", + " id,", + " idPsp,", + " idChannel,", + " idBrokerPsp,", + " cart,", + " idCdi,", + " abi,", + " pspBusinessName,", + " urlPolicyPsp,", + " digitalStamp,", + " digitalStampRestriction,", + " name,", + " description,", + " paymentAmount,", + " minPaymentAmount,", + " maxPaymentAmount,", + " paymentType,", + " touchpoint,", + " type,", + " transferCategoryList,", + " {_etag},", + " lastUpdatedDate = lastUpdatedDateString,", + " insertedDate = insertedDateString,", + " validityDateFrom = validityDateFromString,", + " validityDateTo = validityDateToString", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> selectFields", + "selectFields sink(allowSchemaDrift: true,", + " validateSchema: false,", + " format: 'json',", + " container: 'pagopa-${var.env_short}-itn-observ-az-blob-observability-container',", + " folderPath: 'bundles',", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> afmgecstorage" + ] + sinks = [ + { + description = "Write data to blob storage in json format" + linkedService = { + parameters = {} + referenceName = "afm-gec-${var.env_short}-${var.location_short}-sa-linkedservice" + type = "LinkedServiceReference" } - ] - sources = [ - { - description = "Import data from Analytical Store" - linkedService = { - parameters = {} - referenceName = "afm-gec-${var.env_short}-${var.location_short}-cosmos-linked-service" - type = "LinkedServiceReference" - } - name = "cosmosbundles" + name = "afmgecstorage" + } + ] + sources = [ + { + description = "Import data from Analytical Store" + linkedService = { + parameters = {} + referenceName = "afm-gec-${var.env_short}-${var.location_short}-cosmos-linked-service" + type = "LinkedServiceReference" } - ] - transformations = [ - { - name = "transformDateFormat" - description = "Transform date format from array to string" - }, - { - name = "selectFields" - description = "Select the Date fields with the right format" - } - ] - } + name = "cosmosbundles" + } + ] + transformations = [ + { + name = "transformDateFormat" + description = "Transform date format from array to string" + }, + { + name = "selectFields" + description = "Select the Date fields with the right format" + } + ] + } } -# }) + # }) } } ###################### PDND_CDC_GEC_CIBUNDLES ############################# resource "azapi_resource" "pdnd_cdc_gec_cibundles_dataflow" { - type = "Microsoft.DataFactory/factories/dataflows@2018-06-01" - name = "PDND_CDC_GEC_CIBUNDLES_DataFlow" + type = "Microsoft.DataFactory/factories/dataflows@2018-06-01" + name = "PDND_CDC_GEC_CIBUNDLES_DataFlow" parent_id = data.azurerm_data_factory.qi_data_factory.id #body = jsonencode({ body = { @@ -154,152 +154,152 @@ resource "azapi_resource" "pdnd_cdc_gec_cibundles_dataflow" { name = "PDND_CDC_GEC_DATAFLOW" } type = "MappingDataFlow" - typeProperties = { - #script = "" - scriptLines = [ - "source(output(", - " {_rid} as string,", - " {_ts} as long,", - " ciFiscalCode as string,", - " idBundle as string,", - " type as string,", - " attributes as (id as string, maxPaymentAmount as long, transferCategory as string, transferCategoryRelation as string, insertedDate as long[])[],", - " validityDateFrom as long[],", - " validityDateTo as long[],", - " insertedDate as long[],", - " id as string,", - " {_etag} as string", - " ),", - " allowSchemaDrift: true,", - " validateSchema: false,", - " inferDriftedColumnTypes: true,", - " container: 'cibundles',", - " storeType: 'olap',", - " format: 'document',", - " enableChangeFeed: true,", - " changeFeedStartFromTheBeginning: true,", - " systemColumns: true,", - " captureIntermediateUpdate: false,", - " captureUserDeletes: true,", - " captureTxnTTLDeletes: true,", - " store: 'cosmosDB') ~> cibundles", - "flattenAttribute derive(validityDateFrom = iif(isNull(validityDateFrom), '', toString(validityDateFrom[1]) + '-' + lpad(toString(validityDateFrom[2]), 2, '0') + '-' + lpad(toString(validityDateFrom[3]), 2, '0')),", - " validityDateTo = iif(isNull(validityDateTo), '', toString(validityDateTo[1]) + '-' + lpad(toString(validityDateTo[2]), 2, '0') + '-' + lpad(toString(validityDateTo[3]), 2, '0')),", - " insertedDate = iif(isNull(insertedDate), '', toString(insertedDate[1]) + '-' + lpad(toString(insertedDate[2]), 2, '0') + '-' + lpad(toString(insertedDate[3]), 2, '0') + 'T' + lpad(toString(insertedDate[4]), 2, '0') + ':' + lpad(toString(insertedDate[5]), 2, '0') + ':' + lpad(toString(insertedDate[6]), 2, '0') + '.' + toString(insertedDate[7])),", - " {_etag} = regexReplace({_etag}, '\\\"', ''),", - " attributes = @(id=attributes.id,", - " maxPaymentAmount=attributes.maxPaymentAmount,", - " transferCategory=attributes.transferCategory,", - " transferCategoryRelation=attributes.transferCategoryRelation,", - " insertedDate=iif(isNull(attributes.insertedDate), ", - " '', ", - " toString(attributes.insertedDate[1]) + '-' + ", - " lpad(toString(attributes.insertedDate[2]), 2, '0') + '-' + ", - " lpad(toString(attributes.insertedDate[3]), 2, '0') + 'T' + ", - " lpad(toString(attributes.insertedDate[4]), 2, '0') + ':' + ", - " lpad(toString(attributes.insertedDate[5]), 2, '0') + ':' + ", - " lpad(toString(attributes.insertedDate[6]), 2, '0') + '.' + ", - " toString(attributes.insertedDate[7])))) ~> formatDate", - "aggregateAttribute select(mapColumn(", - " {_rid},", - " {_ts},", - " ciFiscalCode,", - " idBundle,", - " type,", - " attributes,", - " validityDateFrom,", - " validityDateTo,", - " insertedDate,", - " id,", - " {_etag}", - " ),", - " skipDuplicateMapInputs: true,", - " skipDuplicateMapOutputs: true) ~> selctOutputFields", - "cibundles foldDown(unroll(attributes, attributes),", - " mapColumn(", - " {_rid},", - " {_ts},", - " ciFiscalCode,", - " idBundle,", - " type,", - " attributes,", - " validityDateFrom,", - " validityDateTo,", - " insertedDate,", - " id,", - " {_etag}", - " ),", - " skipDuplicateMapInputs: false,", - " skipDuplicateMapOutputs: false) ~> flattenAttribute", - "formatDate aggregate(groupBy({_rid},", - " {_ts},", - " ciFiscalCode,", - " idBundle,", - " type,", - " validityDateFrom,", - " validityDateTo,", - " insertedDate,", - " id,", - " {_etag}),", - " attributes = collect(attributes)) ~> aggregateAttribute", - "selctOutputFields sink(allowSchemaDrift: true,", - " validateSchema: false,", - " format: 'json',", - " container: 'pagopa-${var.env_short}-itn-observ-az-blob-observability-container',", - " folderPath: 'cibundles',", - " skipDuplicateMapInputs: true,", - " skipDuplicateMapOutputs: true) ~> afmgecstorage" - ] - sinks = [ - { - description = "Write data to blob storage in json format" - linkedService = { - parameters = {} - referenceName = "afm-gec-${var.env_short}-${var.location_short}-sa-linkedservice" - type = "LinkedServiceReference" - } - name = "afmgecstorage" + typeProperties = { + #script = "" + scriptLines = [ + "source(output(", + " {_rid} as string,", + " {_ts} as long,", + " ciFiscalCode as string,", + " idBundle as string,", + " type as string,", + " attributes as (id as string, maxPaymentAmount as long, transferCategory as string, transferCategoryRelation as string, insertedDate as long[])[],", + " validityDateFrom as long[],", + " validityDateTo as long[],", + " insertedDate as long[],", + " id as string,", + " {_etag} as string", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " inferDriftedColumnTypes: true,", + " container: 'cibundles',", + " storeType: 'olap',", + " format: 'document',", + " enableChangeFeed: true,", + " changeFeedStartFromTheBeginning: true,", + " systemColumns: true,", + " captureIntermediateUpdate: false,", + " captureUserDeletes: true,", + " captureTxnTTLDeletes: true,", + " store: 'cosmosDB') ~> cibundles", + "flattenAttribute derive(validityDateFrom = iif(isNull(validityDateFrom), '', toString(validityDateFrom[1]) + '-' + lpad(toString(validityDateFrom[2]), 2, '0') + '-' + lpad(toString(validityDateFrom[3]), 2, '0')),", + " validityDateTo = iif(isNull(validityDateTo), '', toString(validityDateTo[1]) + '-' + lpad(toString(validityDateTo[2]), 2, '0') + '-' + lpad(toString(validityDateTo[3]), 2, '0')),", + " insertedDate = iif(isNull(insertedDate), '', toString(insertedDate[1]) + '-' + lpad(toString(insertedDate[2]), 2, '0') + '-' + lpad(toString(insertedDate[3]), 2, '0') + 'T' + lpad(toString(insertedDate[4]), 2, '0') + ':' + lpad(toString(insertedDate[5]), 2, '0') + ':' + lpad(toString(insertedDate[6]), 2, '0') + '.' + toString(insertedDate[7])),", + " {_etag} = regexReplace({_etag}, '\\\"', ''),", + " attributes = @(id=attributes.id,", + " maxPaymentAmount=attributes.maxPaymentAmount,", + " transferCategory=attributes.transferCategory,", + " transferCategoryRelation=attributes.transferCategoryRelation,", + " insertedDate=iif(isNull(attributes.insertedDate), ", + " '', ", + " toString(attributes.insertedDate[1]) + '-' + ", + " lpad(toString(attributes.insertedDate[2]), 2, '0') + '-' + ", + " lpad(toString(attributes.insertedDate[3]), 2, '0') + 'T' + ", + " lpad(toString(attributes.insertedDate[4]), 2, '0') + ':' + ", + " lpad(toString(attributes.insertedDate[5]), 2, '0') + ':' + ", + " lpad(toString(attributes.insertedDate[6]), 2, '0') + '.' + ", + " toString(attributes.insertedDate[7])))) ~> formatDate", + "aggregateAttribute select(mapColumn(", + " {_rid},", + " {_ts},", + " ciFiscalCode,", + " idBundle,", + " type,", + " attributes,", + " validityDateFrom,", + " validityDateTo,", + " insertedDate,", + " id,", + " {_etag}", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> selctOutputFields", + "cibundles foldDown(unroll(attributes, attributes),", + " mapColumn(", + " {_rid},", + " {_ts},", + " ciFiscalCode,", + " idBundle,", + " type,", + " attributes,", + " validityDateFrom,", + " validityDateTo,", + " insertedDate,", + " id,", + " {_etag}", + " ),", + " skipDuplicateMapInputs: false,", + " skipDuplicateMapOutputs: false) ~> flattenAttribute", + "formatDate aggregate(groupBy({_rid},", + " {_ts},", + " ciFiscalCode,", + " idBundle,", + " type,", + " validityDateFrom,", + " validityDateTo,", + " insertedDate,", + " id,", + " {_etag}),", + " attributes = collect(attributes)) ~> aggregateAttribute", + "selctOutputFields sink(allowSchemaDrift: true,", + " validateSchema: false,", + " format: 'json',", + " container: 'pagopa-${var.env_short}-itn-observ-az-blob-observability-container',", + " folderPath: 'cibundles',", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> afmgecstorage" + ] + sinks = [ + { + description = "Write data to blob storage in json format" + linkedService = { + parameters = {} + referenceName = "afm-gec-${var.env_short}-${var.location_short}-sa-linkedservice" + type = "LinkedServiceReference" } - ] - sources = [ - { - description = "Import data from Analytical Store" - linkedService = { - parameters = {} - referenceName = "afm-gec-${var.env_short}-${var.location_short}-cosmos-linked-service" - type = "LinkedServiceReference" - } - name = "cibundles" + name = "afmgecstorage" + } + ] + sources = [ + { + description = "Import data from Analytical Store" + linkedService = { + parameters = {} + referenceName = "afm-gec-${var.env_short}-${var.location_short}-cosmos-linked-service" + type = "LinkedServiceReference" } - ] - transformations = [ - { - name = "formatDate" - description = "Format Date string" - }, - { - name = "selctOutputFields" - description = "select and rename fields" - }, - { - name = "flattenAttribute" - description = "flattern column:\n - id\n - maxPaymentAmount\n - transferCategory\n - transferCategoryRelation\n- insertedDate" - }, - { - name = "aggregateAttribute" - description = "Add attribute columns:\n - id\n - maxPaymentAmount\n - transferCategory\n - transferCategoryRelation\n- insertedDate" - } - ] - } + name = "cibundles" + } + ] + transformations = [ + { + name = "formatDate" + description = "Format Date string" + }, + { + name = "selctOutputFields" + description = "select and rename fields" + }, + { + name = "flattenAttribute" + description = "flattern column:\n - id\n - maxPaymentAmount\n - transferCategory\n - transferCategoryRelation\n- insertedDate" + }, + { + name = "aggregateAttribute" + description = "Add attribute columns:\n - id\n - maxPaymentAmount\n - transferCategory\n - transferCategoryRelation\n- insertedDate" + } + ] + } } -# }) + # }) } } ###################### PDND_CDC_GEC_TOUCHPOINTS_DataFlow ############################# resource "azapi_resource" "pdnd_cdc_gec_touchpoints_dataflow" { - type = "Microsoft.DataFactory/factories/dataflows@2018-06-01" - name = "PDND_CDC_GEC_TOUCHPOINTS_DataFlow" + type = "Microsoft.DataFactory/factories/dataflows@2018-06-01" + name = "PDND_CDC_GEC_TOUCHPOINTS_DataFlow" parent_id = data.azurerm_data_factory.qi_data_factory.id #body = jsonencode({ body = { @@ -310,94 +310,94 @@ resource "azapi_resource" "pdnd_cdc_gec_touchpoints_dataflow" { name = "PDND_CDC_GEC_DATAFLOW" } type = "MappingDataFlow" - typeProperties = { - #script = "" - scriptLines = [ - "source(output(", - " {_rid} as string,", - " {_ts} as long,", - " name as string,", - " id as string,", - " {_etag} as string,", - " creationDate as long[],", - " createdDate as long[]", - " ),", - " allowSchemaDrift: true,", - " validateSchema: false,", - " inferDriftedColumnTypes: true,", - " container: 'touchpoints',", - " storeType: 'olap',", - " format: 'document',", - " enableChangeFeed: true,", - " changeFeedStartFromTheBeginning: true,", - " systemColumns: true,", - " captureIntermediateUpdate: false,", - " captureUserDeletes: true,", - " captureTxnTTLDeletes: true,", - " store: 'cosmosDB') ~> touchpoints", - "touchpoints derive(createdDate = iif(isNull(createdDate), '', toString(createdDate[1]) + '-' + lpad(toString(createdDate[2]), 2, '0') + '-' + lpad(toString(createdDate[3]), 2, '0')),", - " {_etag} = regexReplace({_etag}, '\\\"', '')) ~> formatDateString", - "formatDateString select(mapColumn(", - " {_rid},", - " {_ts},", - " name,", - " id,", - " {_etag},", - " createdDate", - " ),", - " skipDuplicateMapInputs: true,", - " skipDuplicateMapOutputs: true) ~> selectOutputFileds", - "selectOutputFileds sink(allowSchemaDrift: true,", - " validateSchema: false,", - " format: 'json',", - " container: 'pagopa-${var.env_short}-itn-observ-az-blob-observability-container',", - " folderPath: 'touchpoints',", - " truncate: true,", - " skipDuplicateMapInputs: true,", - " skipDuplicateMapOutputs: true) ~> afmgecstorage" - ] - sinks = [ - { - description = "Write data to blob storage in json format" - linkedService = { - parameters = {} - referenceName = "afm-gec-${var.env_short}-${var.location_short}-sa-linkedservice" - type = "LinkedServiceReference" - } - name = "afmgecstorage" + typeProperties = { + #script = "" + scriptLines = [ + "source(output(", + " {_rid} as string,", + " {_ts} as long,", + " name as string,", + " id as string,", + " {_etag} as string,", + " creationDate as long[],", + " createdDate as long[]", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " inferDriftedColumnTypes: true,", + " container: 'touchpoints',", + " storeType: 'olap',", + " format: 'document',", + " enableChangeFeed: true,", + " changeFeedStartFromTheBeginning: true,", + " systemColumns: true,", + " captureIntermediateUpdate: false,", + " captureUserDeletes: true,", + " captureTxnTTLDeletes: true,", + " store: 'cosmosDB') ~> touchpoints", + "touchpoints derive(createdDate = iif(isNull(createdDate), '', toString(createdDate[1]) + '-' + lpad(toString(createdDate[2]), 2, '0') + '-' + lpad(toString(createdDate[3]), 2, '0')),", + " {_etag} = regexReplace({_etag}, '\\\"', '')) ~> formatDateString", + "formatDateString select(mapColumn(", + " {_rid},", + " {_ts},", + " name,", + " id,", + " {_etag},", + " createdDate", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> selectOutputFileds", + "selectOutputFileds sink(allowSchemaDrift: true,", + " validateSchema: false,", + " format: 'json',", + " container: 'pagopa-${var.env_short}-itn-observ-az-blob-observability-container',", + " folderPath: 'touchpoints',", + " truncate: true,", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> afmgecstorage" + ] + sinks = [ + { + description = "Write data to blob storage in json format" + linkedService = { + parameters = {} + referenceName = "afm-gec-${var.env_short}-${var.location_short}-sa-linkedservice" + type = "LinkedServiceReference" } - ] - sources = [ - { - description = "Import data from Analytical Store" - linkedService = { - parameters = {} - referenceName = "afm-gec-${var.env_short}-${var.location_short}-cosmos-linked-service" - type = "LinkedServiceReference" - } - name = "touchpoints" + name = "afmgecstorage" + } + ] + sources = [ + { + description = "Import data from Analytical Store" + linkedService = { + parameters = {} + referenceName = "afm-gec-${var.env_short}-${var.location_short}-cosmos-linked-service" + type = "LinkedServiceReference" } - ] - transformations = [ - { - name = "formatDateString" - description = "Convert the Date format from array to string" - }, - { - name = "selectOutputFileds" - description = "Select output fields" - } - ] - } + name = "touchpoints" + } + ] + transformations = [ + { + name = "formatDateString" + description = "Convert the Date format from array to string" + }, + { + name = "selectOutputFileds" + description = "Select output fields" + } + ] + } } -# }) + # }) } } ###################### PDND_CDC_GEC_PAYMENTTYPES_DataFlow ############################# resource "azapi_resource" "pdnd_cdc_gec_paymenttypes_dataflow" { - type = "Microsoft.DataFactory/factories/dataflows@2018-06-01" - name = "PDND_CDC_GEC_PAYMENTTYPES_DataFlow" + type = "Microsoft.DataFactory/factories/dataflows@2018-06-01" + name = "PDND_CDC_GEC_PAYMENTTYPES_DataFlow" parent_id = data.azurerm_data_factory.qi_data_factory.id #body = jsonencode({ body = { @@ -408,87 +408,87 @@ resource "azapi_resource" "pdnd_cdc_gec_paymenttypes_dataflow" { name = "PDND_CDC_GEC_DATAFLOW" } type = "MappingDataFlow" - typeProperties = { - #script = "" - scriptLines = [ - "source(output(", - " {_rid} as string,", - " {_ts} as long,", - " id as string,", - " name as string,", - " description as string,", - " createdDate as long[],", - " {_etag} as string", - " ),", - " allowSchemaDrift: true,", - " validateSchema: false,", - " inferDriftedColumnTypes: true,", - " container: 'paymenttypes',", - " storeType: 'olap',", - " format: 'document',", - " enableChangeFeed: true,", - " changeFeedStartFromTheBeginning: true,", - " systemColumns: true,", - " captureIntermediateUpdate: false,", - " captureUserDeletes: true,", - " captureTxnTTLDeletes: true,", - " store: 'cosmosDB') ~> paymenttypes", - "paymenttypes derive(createdDate = iif(isNull(createdDate), '', toString(createdDate[1]) + '-' + lpad(toString(createdDate[2]), 2, '0') + '-' + lpad(toString(createdDate[3]), 2, '0')),", - " {_etag} = regexReplace({_etag}, '\\\"', '')) ~> formatDateString", - "formatDateString select(mapColumn(", - " {_rid},", - " {_ts},", - " id,", - " name,", - " description,", - " createdDate,", - " {_etag}", - " ),", - " skipDuplicateMapInputs: true,", - " skipDuplicateMapOutputs: true) ~> selectOutputFileds", - "selectOutputFileds sink(allowSchemaDrift: true,", - " validateSchema: false,", - " format: 'json',", - " container: 'pagopa-${var.env_short}-itn-observ-az-blob-observability-container',", - " folderPath: 'paymenttypes',", - " truncate: true,", - " skipDuplicateMapInputs: true,", - " skipDuplicateMapOutputs: true) ~> afmgecstorage" - ] - sinks = [ - { - description = "Write data to blob storage in json format" - linkedService = { - parameters = {} - referenceName = "afm-gec-${var.env_short}-${var.location_short}-sa-linkedservice" - type = "LinkedServiceReference" - } - name = "afmgecstorage" + typeProperties = { + #script = "" + scriptLines = [ + "source(output(", + " {_rid} as string,", + " {_ts} as long,", + " id as string,", + " name as string,", + " description as string,", + " createdDate as long[],", + " {_etag} as string", + " ),", + " allowSchemaDrift: true,", + " validateSchema: false,", + " inferDriftedColumnTypes: true,", + " container: 'paymenttypes',", + " storeType: 'olap',", + " format: 'document',", + " enableChangeFeed: true,", + " changeFeedStartFromTheBeginning: true,", + " systemColumns: true,", + " captureIntermediateUpdate: false,", + " captureUserDeletes: true,", + " captureTxnTTLDeletes: true,", + " store: 'cosmosDB') ~> paymenttypes", + "paymenttypes derive(createdDate = iif(isNull(createdDate), '', toString(createdDate[1]) + '-' + lpad(toString(createdDate[2]), 2, '0') + '-' + lpad(toString(createdDate[3]), 2, '0')),", + " {_etag} = regexReplace({_etag}, '\\\"', '')) ~> formatDateString", + "formatDateString select(mapColumn(", + " {_rid},", + " {_ts},", + " id,", + " name,", + " description,", + " createdDate,", + " {_etag}", + " ),", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> selectOutputFileds", + "selectOutputFileds sink(allowSchemaDrift: true,", + " validateSchema: false,", + " format: 'json',", + " container: 'pagopa-${var.env_short}-itn-observ-az-blob-observability-container',", + " folderPath: 'paymenttypes',", + " truncate: true,", + " skipDuplicateMapInputs: true,", + " skipDuplicateMapOutputs: true) ~> afmgecstorage" + ] + sinks = [ + { + description = "Write data to blob storage in json format" + linkedService = { + parameters = {} + referenceName = "afm-gec-${var.env_short}-${var.location_short}-sa-linkedservice" + type = "LinkedServiceReference" } - ] - sources = [ - { - description = "Import data from Analytical Store" - linkedService = { - parameters = {} - referenceName = "afm-gec-${var.env_short}-${var.location_short}-cosmos-linked-service" - type = "LinkedServiceReference" - } - name = "paymenttypes" + name = "afmgecstorage" + } + ] + sources = [ + { + description = "Import data from Analytical Store" + linkedService = { + parameters = {} + referenceName = "afm-gec-${var.env_short}-${var.location_short}-cosmos-linked-service" + type = "LinkedServiceReference" } - ] - transformations = [ - { - name = "formatDateString" - description = "Convert the Date format from array to string" - }, - { - name = "selectOutputFileds" - description = "Select output fields" - } - ] - } + name = "paymenttypes" + } + ] + transformations = [ + { + name = "formatDateString" + description = "Convert the Date format from array to string" + }, + { + name = "selectOutputFileds" + description = "Select output fields" + } + ] + } } -# }) + # }) } } \ No newline at end of file diff --git a/src/domains/observability/04_datafactory_dataset.tf b/src/domains/observability/04_datafactory_dataset.tf index a4b7a69d37..ef93e59ea5 100644 --- a/src/domains/observability/04_datafactory_dataset.tf +++ b/src/domains/observability/04_datafactory_dataset.tf @@ -41,12 +41,12 @@ resource "azurerm_data_factory_dataset_json" "afm_gec_bundle_cdc_json" { azure_blob_storage_location { container = "pagopa-${var.env_short}-itn-observ-az-blob-observability-container" - path = "bundles" - filename = "" + path = "bundles" + filename = "" } - encoding = "UTF-8" - folder = local.folder + encoding = "UTF-8" + folder = local.folder annotations = [] } @@ -57,12 +57,12 @@ resource "azurerm_data_factory_dataset_json" "afm_gec_cibundle_cdc_json" { azure_blob_storage_location { container = "pagopa-${var.env_short}-itn-observ-az-blob-observability-container" - path = "cibundles" - filename = "" + path = "cibundles" + filename = "" } - encoding = "UTF-8" - folder = local.folder + encoding = "UTF-8" + folder = local.folder annotations = [] } @@ -73,12 +73,12 @@ resource "azurerm_data_factory_dataset_json" "afm_gec_touchpoints_cdc_json" { azure_blob_storage_location { container = "pagopa-${var.env_short}-itn-observ-az-blob-observability-container" - path = "touchpoints" - filename = "" + path = "touchpoints" + filename = "" } - encoding = "UTF-8" - folder = local.folder + encoding = "UTF-8" + folder = local.folder annotations = [] } @@ -89,12 +89,12 @@ resource "azurerm_data_factory_dataset_json" "afm_gec_paymenttypes_cdc_json" { azure_blob_storage_location { container = "pagopa-${var.env_short}-itn-observ-az-blob-observability-container" - path = "paymenttypes" - filename = "" + path = "paymenttypes" + filename = "" } - encoding = "UTF-8" - folder = local.folder + encoding = "UTF-8" + folder = local.folder annotations = [] } diff --git a/src/domains/observability/99_main.tf b/src/domains/observability/99_main.tf index 28cc1903cf..45f4dc4185 100644 --- a/src/domains/observability/99_main.tf +++ b/src/domains/observability/99_main.tf @@ -15,7 +15,7 @@ terraform { azapi = { source = "azure/azapi" version = "<= 2.0.1" - } + } } backend "azurerm" {} diff --git a/src/domains/observability/README.md b/src/domains/observability/README.md index fd2c439abc..8a6cce15d1 100644 --- a/src/domains/observability/README.md +++ b/src/domains/observability/README.md @@ -6,6 +6,7 @@ | Name | Version | |------|---------| +| [azapi](#requirement\_azapi) | <= 2.0.1 | | [azuread](#requirement\_azuread) | = 2.21.0 | | [azurerm](#requirement\_azurerm) | = 3.53.0 | | [null](#requirement\_null) | = 3.1.1 | @@ -18,15 +19,31 @@ | [apim\_app\_forwarder\_product](#module\_apim\_app\_forwarder\_product) | git::https://github.com/pagopa/terraform-azurerm-v3.git//api_management_product | v6.4.1 | | [app\_forwarder\_app\_service](#module\_app\_forwarder\_app\_service) | git::https://github.com/pagopa/terraform-azurerm-v3.git//app_service | v8.12.2 | | [app\_forwarder\_slot\_staging](#module\_app\_forwarder\_slot\_staging) | git::https://github.com/pagopa/terraform-azurerm-v3.git//app_service_slot | v8.12.2 | +| [eventhub\_namespace\_observability](#module\_eventhub\_namespace\_observability) | git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub | v8.22.0 | +| [eventhub\_namespace\_observability\_gpd](#module\_eventhub\_namespace\_observability\_gpd) | git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub | v8.22.0 | +| [eventhub\_observability\_configuration](#module\_eventhub\_observability\_configuration) | git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration | v8.22.0 | +| [eventhub\_observability\_gpd\_configuration](#module\_eventhub\_observability\_gpd\_configuration) | git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration | v8.22.0 | +| [observability\_sa](#module\_observability\_sa) | git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account | v7.18.0 | +| [observability\_st\_snet](#module\_observability\_st\_snet) | git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet | v6.7.0 | ## Resources | Name | Type | |------|------| +| [azapi_resource.pdnd_cdc_gec_bundles_dataflow](https://registry.terraform.io/providers/azure/azapi/latest/docs/resources/resource) | resource | +| [azapi_resource.pdnd_cdc_gec_cibundles_dataflow](https://registry.terraform.io/providers/azure/azapi/latest/docs/resources/resource) | resource | +| [azapi_resource.pdnd_cdc_gec_paymenttypes_dataflow](https://registry.terraform.io/providers/azure/azapi/latest/docs/resources/resource) | resource | +| [azapi_resource.pdnd_cdc_gec_touchpoints_dataflow](https://registry.terraform.io/providers/azure/azapi/latest/docs/resources/resource) | resource | | [azurerm_api_management_api_version_set.app_forwarder_api](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/api_management_api_version_set) | resource | | [azurerm_api_management_subscription.apim_app_forwarder_subkey](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/api_management_subscription) | resource | | [azurerm_data_factory_custom_dataset.qi_datasets](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_custom_dataset) | resource | | [azurerm_data_factory_custom_dataset.qi_datasets_cosmos](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_custom_dataset) | resource | +| [azurerm_data_factory_dataset_json.afm_gec_bundle_cdc_json](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_dataset_json) | resource | +| [azurerm_data_factory_dataset_json.afm_gec_cibundle_cdc_json](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_dataset_json) | resource | +| [azurerm_data_factory_dataset_json.afm_gec_paymenttypes_cdc_json](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_dataset_json) | resource | +| [azurerm_data_factory_dataset_json.afm_gec_touchpoints_cdc_json](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_dataset_json) | resource | +| [azurerm_data_factory_linked_service_azure_blob_storage.afm_gec_storage_linked_service](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_linked_service_azure_blob_storage) | resource | +| [azurerm_data_factory_linked_service_cosmosdb.afm_gec_cosmosdb_linked_service](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_linked_service_cosmosdb) | resource | | [azurerm_data_factory_linked_service_cosmosdb.cosmos_biz](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_linked_service_cosmosdb) | resource | | [azurerm_data_factory_linked_service_kusto.dataexp_ls](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_linked_service_kusto) | resource | | [azurerm_data_factory_pipeline.pipeline_KPI_FDR_IMPORT_ESITI](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_pipeline) | resource | @@ -37,6 +54,10 @@ | [azurerm_data_factory_pipeline.pipeline_KPI_TPNP](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_pipeline) | resource | | [azurerm_data_factory_pipeline.pipeline_KPI_TPNP_Recupero](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_pipeline) | resource | | [azurerm_data_factory_pipeline.pipeline_KPI_TPSPO_DASPO](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_pipeline) | resource | +| [azurerm_data_factory_pipeline.pipeline_PDND_CDC_GEC_BUNDLES](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_pipeline) | resource | +| [azurerm_data_factory_pipeline.pipeline_PDND_CDC_GEC_CIBUNDLES](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_pipeline) | resource | +| [azurerm_data_factory_pipeline.pipeline_PDND_CDC_GEC_PAYMENTTYPES](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_pipeline) | resource | +| [azurerm_data_factory_pipeline.pipeline_PDND_CDC_GEC_TOUCHPOINTS](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_pipeline) | resource | | [azurerm_data_factory_pipeline.pipeline_PDND_KPI_DASPO](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_pipeline) | resource | | [azurerm_data_factory_pipeline.pipeline_PDND_KPI_LFDR](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_pipeline) | resource | | [azurerm_data_factory_pipeline.pipeline_PDND_KPI_LSPO](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_pipeline) | resource | @@ -50,6 +71,10 @@ | [azurerm_data_factory_trigger_schedule.Trigger_KPI_TNSPO](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | | [azurerm_data_factory_trigger_schedule.Trigger_KPI_TPNP](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | | [azurerm_data_factory_trigger_schedule.Trigger_KPI_TPSPO_DASPO](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | +| [azurerm_data_factory_trigger_schedule.Trigger_PDND_CDC_GEC_BUNDLES](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | +| [azurerm_data_factory_trigger_schedule.Trigger_PDND_CDC_GEC_CIBUNDLES](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | +| [azurerm_data_factory_trigger_schedule.Trigger_PDND_CDC_GEC_PAYMENTTYPES](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | +| [azurerm_data_factory_trigger_schedule.Trigger_PDND_CDC_GEC_TOUCHPOINTS](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | | [azurerm_data_factory_trigger_schedule.Trigger_PDND_KPI_DASPO](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | | [azurerm_data_factory_trigger_schedule.Trigger_PDND_KPI_LFDR](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | | [azurerm_data_factory_trigger_schedule.Trigger_PDND_KPI_LSPO](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | @@ -58,6 +83,7 @@ | [azurerm_data_factory_trigger_schedule.Trigger_PDND_KPI_TPNP](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | | [azurerm_data_factory_trigger_schedule.Trigger_PDND_KPI_WAFDR](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | | [azurerm_data_factory_trigger_schedule.Trigger_PDND_KPI_WPNFDR](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/data_factory_trigger_schedule) | resource | +| [azurerm_eventhub_namespace_authorization_rule.cdc_connection_string](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/eventhub_namespace_authorization_rule) | resource | | [azurerm_key_vault_secret.apim_app_forwarder_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.certificate_crt_app_forwarder_s](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.certificate_key_app_forwarder_s](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/key_vault_secret) | resource | @@ -65,9 +91,17 @@ | [azurerm_kusto_database.re_db](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/kusto_database) | resource | | [azurerm_kusto_database_principal_assignment.qi_principal_assignment](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/kusto_database_principal_assignment) | resource | | [azurerm_kusto_eventhub_data_connection.eventhub_connection_for_re_event](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/kusto_eventhub_data_connection) | resource | +| [azurerm_private_endpoint.observability_storage_private_endpoint](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/private_endpoint) | resource | +| [azurerm_resource_group.eventhub_observability_rg](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/resource_group) | resource | +| [azurerm_resource_group.st_observability_rg](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/resource_group) | resource | +| [azurerm_storage_container.blob-observability-st](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/storage_container) | resource | +| [azurerm_subnet.eventhub_observability_gpd_snet](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/subnet) | resource | +| [azurerm_subnet.eventhub_observability_snet](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/subnet) | resource | | [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/client_config) | data source | | [azurerm_container_registry.acr](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/container_registry) | data source | +| [azurerm_cosmosdb_account.afm_cosmos_account](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/cosmosdb_account) | data source | | [azurerm_cosmosdb_account.bizevent_cosmos_account](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/cosmosdb_account) | data source | +| [azurerm_data_factory.obeserv_data_factory](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/data_factory) | data source | | [azurerm_data_factory.qi_data_factory](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/data_factory) | data source | | [azurerm_data_factory.qi_data_factory_cosmos](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/data_factory) | data source | | [azurerm_eventhub.pagopa-evh-ns03_nodo-dei-pagamenti-re_nodo-dei-pagamenti-re](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/eventhub) | data source | @@ -75,11 +109,20 @@ | [azurerm_key_vault.kv_shared](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/key_vault) | data source | | [azurerm_key_vault_secret.certificate_crt_app_forwarder](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/key_vault_secret) | data source | | [azurerm_key_vault_secret.certificate_key_app_forwarder](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/key_vault_secret) | data source | +| [azurerm_monitor_action_group.email](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/monitor_action_group) | data source | +| [azurerm_monitor_action_group.slack](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/monitor_action_group) | data source | +| [azurerm_private_dns_zone.eventhub](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/private_dns_zone) | data source | +| [azurerm_private_dns_zone.storage](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/private_dns_zone) | data source | | [azurerm_resource_group.monitor_rg](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/resource_group) | data source | +| [azurerm_resource_group.rg_event_private_dns_zone](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/resource_group) | data source | | [azurerm_resource_group.rg_node_forwarder](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/resource_group) | data source | +| [azurerm_resource_group.rg_vnet_italy](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/resource_group) | data source | +| [azurerm_storage_account.observ_storage_account](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/storage_account) | data source | | [azurerm_subnet.subnet_apim](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/subnet) | data source | | [azurerm_subnet.subnet_node_forwarder](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/subnet) | data source | | [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/subscription) | data source | +| [azurerm_virtual_network.vnet](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/virtual_network) | data source | +| [azurerm_virtual_network.vnet_italy](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/data-sources/virtual_network) | data source | ## Inputs @@ -87,19 +130,44 @@ |------|-------------|------|---------|:--------:| | [apim\_dns\_zone\_prefix](#input\_apim\_dns\_zone\_prefix) | The dns subdomain for apim. | `string` | `null` | no | | [app\_forwarder\_enabled](#input\_app\_forwarder\_enabled) | Enable app\_forwarder | `bool` | `false` | no | -| [dexp\_db](#input\_dexp\_db) | n/a | 
object({
    enable             = bool
    hot_cache_period   = string
    soft_delete_period = string
  })
| n/a | yes | -| [dexp\_params](#input\_dexp\_params) | n/a | 
object({
    enabled = bool
    sku = object({
      name     = string
      capacity = number
    })
    autoscale = object({
      enabled       = bool
      min_instances = number
      max_instances = number
    })
    public_network_access_enabled = bool
    double_encryption_enabled     = bool
    disk_encryption_enabled       = bool
    purge_enabled                 = bool
  })
| n/a | yes | -| [dexp\_re\_db\_linkes\_service](#input\_dexp\_re\_db\_linkes\_service) | n/a | 
object({
    enable = bool
  })
| n/a | yes | +| [cidr\_subnet\_observability\_evh](#input\_cidr\_subnet\_observability\_evh) | Address prefixes evh | `list(string)` | n/a | yes | +| [cidr\_subnet\_observability\_gpd\_evh](#input\_cidr\_subnet\_observability\_gpd\_evh) | Address prefixes evh | `list(string)` | n/a | yes | +| [cidr\_subnet\_observability\_storage](#input\_cidr\_subnet\_observability\_storage) | Storage address space | `list(string)` | `null` | no | +| [dexp\_db](#input\_dexp\_db) | n/a | 
object({
    enable             = bool
    hot_cache_period   = string
    soft_delete_period = string
  })
| n/a | yes | +| [dexp\_params](#input\_dexp\_params) | n/a | 
object({
    enabled = bool
    sku = object({
      name     = string
      capacity = number
    })
    autoscale = object({
      enabled       = bool
      min_instances = number
      max_instances = number
    })
    public_network_access_enabled = bool
    double_encryption_enabled     = bool
    disk_encryption_enabled       = bool
    purge_enabled                 = bool
  })
| n/a | yes | +| [dexp\_re\_db\_linkes\_service](#input\_dexp\_re\_db\_linkes\_service) | n/a | 
object({
    enable = bool
  })
| n/a | yes | | [domain](#input\_domain) | n/a | `string` | n/a | yes | +| [ehns\_alerts\_enabled](#input\_ehns\_alerts\_enabled) | Event hub alerts enabled? | `bool` | n/a | yes | +| [ehns\_auto\_inflate\_enabled](#input\_ehns\_auto\_inflate\_enabled) | Is Auto Inflate enabled for the EventHub Namespace? | `bool` | n/a | yes | +| [ehns\_capacity](#input\_ehns\_capacity) | Specifies the Capacity / Throughput Units for a Standard SKU namespace. | `number` | n/a | yes | +| [ehns\_maximum\_throughput\_units](#input\_ehns\_maximum\_throughput\_units) | Specifies the maximum number of throughput units when Auto Inflate is Enabled | `number` | n/a | yes | +| [ehns\_metric\_alerts](#input\_ehns\_metric\_alerts) | Map of name = criteria objects | 
map(object({
    # criteria.*.aggregation to be one of [Average Count Minimum Maximum Total]
    aggregation = string
    metric_name = string
    description = string
    # criteria.0.operator to be one of [Equals NotEquals GreaterThan GreaterThanOrEqual LessThan LessThanOrEqual]
    operator  = string
    threshold = number
    # Possible values are PT1M, PT5M, PT15M, PT30M and PT1H
    frequency = string
    # Possible values are PT1M, PT5M, PT15M, PT30M, PT1H, PT6H, PT12H and P1D.
    window_size = string

    dimension = list(object(
      {
        name     = string
        operator = string
        values   = list(string)
      }
    ))
  }))
| `{}` | no | +| [ehns\_metric\_alerts\_gpd](#input\_ehns\_metric\_alerts\_gpd) | Map of name = criteria objects | 
map(object({
    # criteria.*.aggregation to be one of [Average Count Minimum Maximum Total]
    aggregation = string
    metric_name = string
    description = string
    # criteria.0.operator to be one of [Equals NotEquals GreaterThan GreaterThanOrEqual LessThan LessThanOrEqual]
    operator  = string
    threshold = number
    # Possible values are PT1M, PT5M, PT15M, PT30M and PT1H
    frequency = string
    # Possible values are PT1M, PT5M, PT15M, PT30M, PT1H, PT6H, PT12H and P1D.
    window_size = string

    dimension = list(object(
      {
        name     = string
        operator = string
        values   = list(string)
      }
    ))
  }))
| `{}` | no | +| [ehns\_private\_endpoint\_is\_present](#input\_ehns\_private\_endpoint\_is\_present) | (Required) create private endpoint to the event hubs | `bool` | n/a | yes | +| [ehns\_public\_network\_access](#input\_ehns\_public\_network\_access) | (Required) enables public network access to the event hubs | `bool` | n/a | yes | +| [ehns\_sku\_name](#input\_ehns\_sku\_name) | Defines which tier to use. | `string` | n/a | yes | +| [ehns\_zone\_redundant](#input\_ehns\_zone\_redundant) | Specifies if the EventHub Namespace should be Zone Redundant (created across Availability Zones). | `bool` | n/a | yes | +| [enable\_sa\_backup](#input\_enable\_sa\_backup) | (Optional) enables storage account point in time recovery | `bool` | `false` | no | | [env](#input\_env) | n/a | `string` | n/a | yes | | [env\_short](#input\_env\_short) | n/a | `string` | n/a | yes | +| [eventhubs](#input\_eventhubs) | A list of event hubs to add to namespace. | 
list(object({
    name              = string
    partitions        = number
    message_retention = number
    consumers         = list(string)
    keys = list(object({
      name   = string
      listen = bool
      send   = bool
      manage = bool
    }))
  }))
| `[]` | no | +| [eventhubs\_gpd](#input\_eventhubs\_gpd) | A list of event hubs to add to namespace. | 
list(object({
    name              = string
    partitions        = number
    message_retention = number
    consumers         = list(string)
    keys = list(object({
      name   = string
      listen = bool
      send   = bool
      manage = bool
    }))
  }))
| `[]` | no | | [external\_domain](#input\_external\_domain) | Domain for delegation | `string` | `null` | no | | [instance](#input\_instance) | One of beta, prod01, prod02 | `string` | n/a | yes | | [location](#input\_location) | One of westeurope, northeurope | `string` | n/a | yes | +| [location\_itn](#input\_location\_itn) | italynorth | `string` | n/a | yes | | [location\_short](#input\_location\_short) | One of wue, neu | `string` | n/a | yes | +| [location\_short\_itn](#input\_location\_short\_itn) | itn | `string` | n/a | yes | | [monitor\_resource\_group\_name](#input\_monitor\_resource\_group\_name) | Monitor resource group name | `string` | n/a | yes | +| [observability\_sa\_advanced\_threat\_protection](#input\_observability\_sa\_advanced\_threat\_protection) | Enable contract threat advanced protection | `bool` | `false` | no | +| [observability\_sa\_backup\_retention\_days](#input\_observability\_sa\_backup\_retention\_days) | Number of days to retain backups. | `number` | `0` | no | +| [observability\_sa\_delete\_after\_last\_access](#input\_observability\_sa\_delete\_after\_last\_access) | Number of days since modification to blob before deleting | `number` | `3650` | no | +| [observability\_sa\_delete\_retention\_days](#input\_observability\_sa\_delete\_retention\_days) | Number of days to retain deleted. | `number` | `0` | no | +| [observability\_sa\_tier\_to\_cool\_after\_last\_access](#input\_observability\_sa\_tier\_to\_cool\_after\_last\_access) | Number of days since last access to blob before moving to cool tier | `number` | `183` | no | +| [observability\_storage\_account\_replication\_type](#input\_observability\_storage\_account\_replication\_type) | (Optional) observability datastore storage account replication type | `string` | `"LRS"` | no | +| [observability\_tier\_to\_archive\_after\_days\_since\_last\_access\_time\_greater\_than](#input\_observability\_tier\_to\_archive\_after\_days\_since\_last\_access\_time\_greater\_than) | Number of days since last access to blob before moving to archive tier | `number` | `730` | no | | [prefix](#input\_prefix) | n/a | `string` | n/a | yes | -| [tags](#input\_tags) | n/a | `map(any)` | 
{
  "CreatedBy": "Terraform"
}
| no | +| [tags](#input\_tags) | n/a | `map(any)` | 
{
  "CreatedBy": "Terraform"
}
| no | ## Outputs From 02d6a6e3ee67126e6457d43288380c13fa5b400e Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Wed, 13 Nov 2024 14:07:09 +0100 Subject: [PATCH 31/55] fix --- src/domains/gps-app/05_debezium_connect.tf | 2 +- .../gps-app/yaml/postgres-connector.yaml | 4 +- src/domains/gps-common/01_network.tf | 2 +- src/domains/gps-common/02_security.tf | 98 ++++++++++ src/domains/gps-common/03_postgresql_gpd.tf | 4 +- .../gps-common/03_postgresql_replica.tf | 2 +- src/domains/gps-common/10_github_identity.tf | 1 + src/domains/gps-common/README.md | 64 ++++--- .../secret/weu-dev/noedit_secret_enc.json | 5 +- .../observability/03_gpd_ingestion_sa.tf | 36 ++++ src/domains/observability/99_variables.tf | 23 +++ src/domains/observability/README.md | 7 +- .../observability/env/dev/terraform.tfvars | 169 ++++++------------ .../observability/gpd_evh_create__az.sh | 104 +++++------ .../observability/gpd_evh_delete__az.sh | 50 +----- src/psql/flyway_gpd.sh | 2 +- ...V2__CDC_USER.sql => how2_add_cdc_user.sql} | 0 17 files changed, 318 insertions(+), 255 deletions(-) create mode 100644 src/domains/observability/03_gpd_ingestion_sa.tf rename src/psql/migrations/{DEV-pagoPA/apd/V2__CDC_USER.sql => how2_add_cdc_user.sql} (100%) diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index 1905ee476f..b7201357cd 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -75,7 +75,7 @@ locals { postgres_port = 5432 postgres_db_name = var.postgres_db_name - postgres_topic_prefix = "azcligpd" + postgres_topic_prefix = "cdc-raw-auto" postgres_username = data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value postgres_password = data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value tasks_max = var.tasks_max diff --git a/src/domains/gps-app/yaml/postgres-connector.yaml b/src/domains/gps-app/yaml/postgres-connector.yaml index d87528b794..54aacf7c1f 100644 --- a/src/domains/gps-app/yaml/postgres-connector.yaml +++ b/src/domains/gps-app/yaml/postgres-connector.yaml @@ -22,7 +22,7 @@ spec: database.dbname: ${postgres_db_name} topic.prefix: ${postgres_topic_prefix} schema.include.list: "apd" - table.include.list: "apd.payment_option,apd.payment_option_metadata,apd.payment_position,apd.transfer,apd.transfer_metadata" + table.include.list: "apd.payment_option,apd.payment_position,apd.transfer" plugin.name: "pgoutput" publication.autocreate.mode: "disabled" # shall be create before - # publication.autocreate.mode: "filtered" + # publication.autocreate.mode: "filtered" # create it diff --git a/src/domains/gps-common/01_network.tf b/src/domains/gps-common/01_network.tf index 0a6504542e..a7805d628f 100644 --- a/src/domains/gps-common/01_network.tf +++ b/src/domains/gps-common/01_network.tf @@ -1,6 +1,6 @@ # Azure Storage subnet module "storage_account_snet" { - source = "./.terraform/modules/__v3__/subnet" + source = "./.terraform/modules/__v3__/subnet" name = "${local.project}-storage-account-snet" address_prefixes = var.gpd_sftp_cidr_subnet_gpd_storage_account resource_group_name = local.vnet_resource_group_name diff --git a/src/domains/gps-common/02_security.tf b/src/domains/gps-common/02_security.tf index 7e76d3cc6c..5cdc899000 100644 --- a/src/domains/gps-common/02_security.tf +++ b/src/domains/gps-common/02_security.tf @@ -491,3 +491,101 @@ resource "azurerm_key_vault_secret" "elastic_otel_token_header" { ] } } + +# ####################### +# CDC GPD config secrets +# ####################### + + +data "azurerm_storage_account" "gpd_ingestion_sa" { + name = "pagopa${var.env_short}gpdingestsa" + resource_group_name = "pagopa-${var.env_short}-itn-observ-gpd-rg" +} + +resource "azurerm_key_vault_secret" "azure_web_jobs_storage_kv" { + name = "AzureWebJobsStorage-gdp-ingestion" + value = data.azurerm_storage_account.gpd_ingestion_sa.primary_connection_string + content_type = "text/plain" + key_vault_id = module.key_vault.id +} + + +# CDC GDP in eventhub +data "azurerm_eventhub_authorization_rule" "cdc-raw-auto_apd_payment_option-rx" { + name = "cdc-raw-auto.apd.payment_option-rx" + namespace_name = "pagopa-${var.env_short}-itn-observ-gpd-evh" + eventhub_name = "cdc-raw-auto.apd.payment_option" + resource_group_name = "pagopa-${var.env_short}-itn-observ-evh-rg" +} +resource "azurerm_key_vault_secret" "cdc-raw-auto_apd_payment_option-rx_kv" { + name = "payment-option-topic-input-conn-string" + value = data.azurerm_eventhub_authorization_rule.cdc-raw-auto_apd_payment_option-rx.primary_connection_string + content_type = "text/plain" + key_vault_id = module.key_vault.id +} +data "azurerm_eventhub_authorization_rule" "cdc-raw-auto_apd_payment_position-rx" { + name = "cdc-raw-auto.apd.payment_position-rx" + namespace_name = "pagopa-${var.env_short}-itn-observ-gpd-evh" + eventhub_name = "cdc-raw-auto.apd.payment_position" + resource_group_name = "pagopa-${var.env_short}-itn-observ-evh-rg" +} +resource "azurerm_key_vault_secret" "cdc-raw-auto_apd_payment_position-rx_kv" { + name = "payment-position-topic-input-conn-string" + value = data.azurerm_eventhub_authorization_rule.cdc-raw-auto_apd_payment_position-rx.primary_connection_string + content_type = "text/plain" + key_vault_id = module.key_vault.id +} +data "azurerm_eventhub_authorization_rule" "cdc-raw-auto_apd_transfer-rx" { + name = "cdc-raw-auto.apd.transfer-rx" + namespace_name = "pagopa-${var.env_short}-itn-observ-gpd-evh" + eventhub_name = "cdc-raw-auto.apd.transfer" + resource_group_name = "pagopa-${var.env_short}-itn-observ-evh-rg" +} +resource "azurerm_key_vault_secret" "cdc-raw-auto_apd_transfer-rx_kv" { + name = "transfer-topic-input-conn-string" + value = data.azurerm_eventhub_authorization_rule.cdc-raw-auto_apd_transfer-rx.primary_connection_string + content_type = "text/plain" + key_vault_id = module.key_vault.id +} +# CDC GDP out eventhub +data "azurerm_eventhub_authorization_rule" "gpd_ingestion_apd_payment_option_tx" { + name = "gpd-ingestion.apd.payment_option-tx" + namespace_name = "pagopa-${var.env_short}-itn-observ-gpd-evh" + eventhub_name = "gpd-ingestion.apd.payment_option" + resource_group_name = "pagopa-${var.env_short}-itn-observ-evh-rg" +} + +resource "azurerm_key_vault_secret" "gpd_ingestion_apd_payment_option_tx_kv" { + name = "payment-option-topic-output-conn-string" + value = data.azurerm_eventhub_authorization_rule.gpd_ingestion_apd_payment_option_tx.primary_connection_string + content_type = "text/plain" + key_vault_id = module.key_vault.id +} + +data "azurerm_eventhub_authorization_rule" "gpd_ingestion_apd_payment_position_tx" { + name = "gpd-ingestion.apd.payment_position-tx" + namespace_name = "pagopa-${var.env_short}-itn-observ-gpd-evh" + eventhub_name = "gpd-ingestion.apd.payment_position" + resource_group_name = "pagopa-${var.env_short}-itn-observ-evh-rg" +} + +resource "azurerm_key_vault_secret" "gpd_ingestion_apd_payment_position_tx_kv" { + name = "payment-position-topic-output-conn-string" + value = data.azurerm_eventhub_authorization_rule.gpd_ingestion_apd_payment_position_tx.primary_connection_string + content_type = "text/plain" + key_vault_id = module.key_vault.id +} + +data "azurerm_eventhub_authorization_rule" "gpd_ingestion_apd_payment_option_transfer_tx" { + name = "gpd-ingestion.apd.transfer-tx" + namespace_name = "pagopa-${var.env_short}-itn-observ-gpd-evh" + eventhub_name = "gpd-ingestion.apd.transfer" + resource_group_name = "pagopa-${var.env_short}-itn-observ-evh-rg" +} + +resource "azurerm_key_vault_secret" "gpd_ingestion_apd_payment_option_transfer_tx_kv" { + name = "transfer-topic-output-conn-string" + value = data.azurerm_eventhub_authorization_rule.gpd_ingestion_apd_payment_option_transfer_tx.primary_connection_string + content_type = "text/plain" + key_vault_id = module.key_vault.id +} \ No newline at end of file diff --git a/src/domains/gps-common/03_postgresql_gpd.tf b/src/domains/gps-common/03_postgresql_gpd.tf index 4bf2879213..ab280b4cb6 100644 --- a/src/domains/gps-common/03_postgresql_gpd.tf +++ b/src/domains/gps-common/03_postgresql_gpd.tf @@ -15,7 +15,7 @@ data "azurerm_key_vault_secret" "pgres_admin_pwd" { resource "azurerm_resource_group" "flex_data" { count = 1 # forced - + name = format("%s-pgres-flex-rg", local.product) location = var.location @@ -59,7 +59,7 @@ data "azurerm_private_dns_zone" "postgres" { # https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-compare-single-server-flexible-server module "postgres_flexible_server_private" { # private only into UAT and PROD env source = "./.terraform/modules/__v3__/postgres_flexible_server" - count = 1 # forced + count = 1 # forced name = format("%s-gpd-pgflex", local.product) diff --git a/src/domains/gps-common/03_postgresql_replica.tf b/src/domains/gps-common/03_postgresql_replica.tf index 7d51257f9c..2856503b4e 100644 --- a/src/domains/gps-common/03_postgresql_replica.tf +++ b/src/domains/gps-common/03_postgresql_replica.tf @@ -2,7 +2,7 @@ ## Postgres Flexible Server subnet module "postgres_flexible_snet_replica" { count = var.geo_replica_enabled ? 1 : 0 - source = "./.terraform/modules/__v3__/subnet" + source = "./.terraform/modules/__v3__/subnet" name = "${local.project_replica}-pgres-flexible-snet" address_prefixes = var.geo_replica_cidr_subnet_postgresql resource_group_name = data.azurerm_resource_group.rg_vnet.name diff --git a/src/domains/gps-common/10_github_identity.tf b/src/domains/gps-common/10_github_identity.tf index 54ac2b29ba..2620b7e0d2 100644 --- a/src/domains/gps-common/10_github_identity.tf +++ b/src/domains/gps-common/10_github_identity.tf @@ -18,6 +18,7 @@ locals { "pagopa-gpd-reporting-batch", "pagopa-gpd-reporting-analysis", "pagopa-gpd-reporting-service", + "pagopa-gpd-ingestion-manager" ] federations_01 = [ diff --git a/src/domains/gps-common/README.md b/src/domains/gps-common/README.md index 686bd2a44e..ba7cdcdf60 100644 --- a/src/domains/gps-common/README.md +++ b/src/domains/gps-common/README.md @@ -1,36 +1,35 @@ - + ## Requirements | Name | Version | |------|---------| | [azapi](#requirement\_azapi) | <= 1.13.1 | -| [azuread](#requirement\_azuread) | <= 2.21.0 | -| [azurerm](#requirement\_azurerm) | <= 3.53.0 | -| [null](#requirement\_null) | <= 3.2.1 | +| [azuread](#requirement\_azuread) | <= 2.47.0 | +| [azurerm](#requirement\_azurerm) | <= 3.107.0 | +| [null](#requirement\_null) | <= 3.2.2 | ## Modules | Name | Source | Version | |------|--------|---------| -| [flows](#module\_flows) | git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account | v7.18.0 | -| [gpd\_cosmosdb\_containers](#module\_gpd\_cosmosdb\_containers) | git::https://github.com/pagopa/terraform-azurerm-v3.git//cosmosdb_sql_container | v6.4.1 | -| [gpd\_cosmosdb\_database](#module\_gpd\_cosmosdb\_database) | git::https://github.com/pagopa/terraform-azurerm-v3.git//cosmosdb_sql_database | v6.4.1 | -| [gpd\_payments\_cosmosdb\_account](#module\_gpd\_payments\_cosmosdb\_account) | git::https://github.com/pagopa/terraform-azurerm-v3.git//cosmosdb_account | v7.0.0 | -| [gpd\_sa\_sftp](#module\_gpd\_sa\_sftp) | git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account | v7.18.0 | -| [gps\_cosmosdb\_account](#module\_gps\_cosmosdb\_account) | git::https://github.com/pagopa/terraform-azurerm-v3.git//cosmosdb_account | v6.4.1 | -| [gps\_cosmosdb\_containers](#module\_gps\_cosmosdb\_containers) | git::https://github.com/pagopa/terraform-azurerm-v3.git//cosmosdb_sql_container | v6.4.1 | -| [gps\_cosmosdb\_database](#module\_gps\_cosmosdb\_database) | git::https://github.com/pagopa/terraform-azurerm-v3.git//cosmosdb_sql_database | v6.4.1 | -| [gps\_cosmosdb\_snet](#module\_gps\_cosmosdb\_snet) | git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet | v6.6.1 | -| [identity\_cd\_01](#module\_identity\_cd\_01) | github.com/pagopa/terraform-azurerm-v3//github_federated_identity | v7.45.0 | -| [key\_vault](#module\_key\_vault) | git::https://github.com/pagopa/terraform-azurerm-v3.git//key_vault | v6.4.1 | -| [postgres\_flexible\_server\_private](#module\_postgres\_flexible\_server\_private) | git::https://github.com/pagopa/terraform-azurerm-v3//postgres_flexible_server | v7.23.0 | -| [postgres\_flexible\_snet](#module\_postgres\_flexible\_snet) | git::https://github.com/pagopa/terraform-azurerm-v3//subnet | v6.11.2 | -| [postgres\_flexible\_snet\_replica](#module\_postgres\_flexible\_snet\_replica) | git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet | v7.22.0 | -| [postgresql](#module\_postgresql) | git::https://github.com/pagopa/terraform-azurerm-v3//postgresql_server | v6.11.2 | -| [postgresql\_gpd\_replica\_db](#module\_postgresql\_gpd\_replica\_db) | git::https://github.com/pagopa/terraform-azurerm-v3.git//postgres_flexible_server_replica | v7.22.0 | -| [postgresql\_snet](#module\_postgresql\_snet) | git::https://github.com/pagopa/terraform-azurerm-v3//subnet | v6.11.2 | -| [storage\_account\_snet](#module\_storage\_account\_snet) | git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet | v6.2.1 | +| [\_\_v3\_\_](#module\_\_\_v3\_\_) | git::https://github.com/pagopa/terraform-azurerm-v3 | v8.52.0 | +| [flows](#module\_flows) | ./.terraform/modules/__v3__/storage_account | n/a | +| [gpd\_cosmosdb\_containers](#module\_gpd\_cosmosdb\_containers) | ./.terraform/modules/__v3__/cosmosdb_sql_container | n/a | +| [gpd\_cosmosdb\_database](#module\_gpd\_cosmosdb\_database) | ./.terraform/modules/__v3__/cosmosdb_sql_database | n/a | +| [gpd\_payments\_cosmosdb\_account](#module\_gpd\_payments\_cosmosdb\_account) | ./.terraform/modules/__v3__/cosmosdb_account | n/a | +| [gpd\_sa\_sftp](#module\_gpd\_sa\_sftp) | ./.terraform/modules/__v3__/storage_account | n/a | +| [gps\_cosmosdb\_account](#module\_gps\_cosmosdb\_account) | ./.terraform/modules/__v3__/cosmosdb_account | n/a | +| [gps\_cosmosdb\_containers](#module\_gps\_cosmosdb\_containers) | ./.terraform/modules/__v3__/cosmosdb_sql_container | n/a | +| [gps\_cosmosdb\_database](#module\_gps\_cosmosdb\_database) | ./.terraform/modules/__v3__/cosmosdb_sql_database | n/a | +| [gps\_cosmosdb\_snet](#module\_gps\_cosmosdb\_snet) | ./.terraform/modules/__v3__/subnet | n/a | +| [identity\_cd\_01](#module\_identity\_cd\_01) | ./.terraform/modules/__v3__/github_federated_identity | n/a | +| [key\_vault](#module\_key\_vault) | ./.terraform/modules/__v3__/key_vault | n/a | +| [postgres\_flexible\_server\_private](#module\_postgres\_flexible\_server\_private) | ./.terraform/modules/__v3__/postgres_flexible_server | n/a | +| [postgres\_flexible\_snet](#module\_postgres\_flexible\_snet) | ./.terraform/modules/__v3__/subnet | n/a | +| [postgres\_flexible\_snet\_replica](#module\_postgres\_flexible\_snet\_replica) | ./.terraform/modules/__v3__/subnet | n/a | +| [postgresql\_gpd\_replica\_db](#module\_postgresql\_gpd\_replica\_db) | ./.terraform/modules/__v3__/postgres_flexible_server_replica | n/a | +| [storage\_account\_snet](#module\_storage\_account\_snet) | ./.terraform/modules/__v3__/subnet | n/a | ## Resources @@ -48,6 +47,10 @@ | [azurerm_key_vault_access_policy.azdevops_iac_policy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | | [azurerm_key_vault_access_policy.gha_iac_managed_identities](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_access_policy) | resource | | [azurerm_key_vault_secret.ai_connection_string](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.azure_web_jobs_storage_kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.cdc-raw-auto_apd_payment_option-rx_kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.cdc-raw-auto_apd_payment_position-rx_kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.cdc-raw-auto_apd_transfer-rx_kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.cosmos_gps_pkey](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.db_url](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.elastic_otel_token_header](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | @@ -61,6 +64,9 @@ | [azurerm_key_vault_secret.gpd_donations_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.gpd_gpd_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.gpd_gps_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.gpd_ingestion_apd_payment_option_transfer_tx_kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.gpd_ingestion_apd_payment_option_tx_kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | +| [azurerm_key_vault_secret.gpd_ingestion_apd_payment_position_tx_kv](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.gpd_iuv_generator_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.gpd_node_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.gpd_payments_rest_subscription_key](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | @@ -78,7 +84,6 @@ | [azurerm_key_vault_secret.pgres_admin_pwd](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_key_vault_secret.storage_reporting_connection_string](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_monitor_scheduled_query_rules_alert.payments_gpd_inconsistency_error](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/monitor_scheduled_query_rules_alert) | resource | -| [azurerm_postgresql_database.apd_db](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/postgresql_database) | resource | | [azurerm_postgresql_flexible_server_configuration.apd_db_flex_ignore_startup_parameters](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/postgresql_flexible_server_configuration) | resource | | [azurerm_postgresql_flexible_server_configuration.apd_db_flex_max_connection](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/postgresql_flexible_server_configuration) | resource | | [azurerm_postgresql_flexible_server_configuration.apd_db_flex_max_worker_process](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/postgresql_flexible_server_configuration) | resource | @@ -114,6 +119,12 @@ | [azuread_service_principal.iac_principal](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/service_principal) | data source | | [azurerm_application_insights.application_insights](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/application_insights) | data source | | [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | +| [azurerm_eventhub_authorization_rule.cdc-raw-auto_apd_payment_option-rx](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | +| [azurerm_eventhub_authorization_rule.cdc-raw-auto_apd_payment_position-rx](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | +| [azurerm_eventhub_authorization_rule.cdc-raw-auto_apd_transfer-rx](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | +| [azurerm_eventhub_authorization_rule.gpd_ingestion_apd_payment_option_transfer_tx](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | +| [azurerm_eventhub_authorization_rule.gpd_ingestion_apd_payment_option_tx](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | +| [azurerm_eventhub_authorization_rule.gpd_ingestion_apd_payment_position_tx](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/eventhub_authorization_rule) | data source | | [azurerm_key_vault_secret.pgres_admin_login](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | | [azurerm_key_vault_secret.pgres_admin_pwd](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault_secret) | data source | | [azurerm_kubernetes_cluster.aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/kubernetes_cluster) | data source | @@ -130,6 +141,7 @@ | [azurerm_resource_group.identity_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | | [azurerm_resource_group.monitor_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | | [azurerm_resource_group.rg_vnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | +| [azurerm_storage_account.gpd_ingestion_sa](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/storage_account) | data source | | [azurerm_subnet.aks_snet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | | [azurerm_subnet.aks_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | | [azurerm_subnet.azdo_snet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | @@ -146,6 +158,7 @@ | [cidr\_subnet\_gpd\_payments\_cosmosdb](#input\_cidr\_subnet\_gpd\_payments\_cosmosdb) | Cosmos DB gpd payments address space | `list(string)` | `null` | no | | [cidr\_subnet\_gps\_cosmosdb](#input\_cidr\_subnet\_gps\_cosmosdb) | Cosmos DB address space | `list(string)` | `null` | no | | [cidr\_subnet\_pg\_flex\_dbms](#input\_cidr\_subnet\_pg\_flex\_dbms) | Postgres Flexible Server network address space. | `list(string)` | n/a | yes | +| [cidr\_subnet\_pg\_singleser](#input\_cidr\_subnet\_pg\_singleser) | Postgres Single Server network address space. | `list(string)` | `[]` | no | | [cosmos\_gpd\_payments\_db\_params](#input\_cosmos\_gpd\_payments\_db\_params) | n/a | 
object({
    kind           = string
    capabilities   = list(string)
    offer_type     = string
    server_version = string
    consistency_policy = object({
      consistency_level       = string
      max_interval_in_seconds = number
      max_staleness_prefix    = number
    })
    main_geo_location_zone_redundant = bool
    enable_free_tier                 = bool
    additional_geo_locations = list(object({
      location          = string
      failover_priority = number
      zone_redundant    = bool
    }))
    private_endpoint_enabled          = bool
    public_network_access_enabled     = bool
    is_virtual_network_filter_enabled = bool
    backup_continuous_enabled         = bool
    payments_receipts_table = object({
      autoscale  = bool
      throughput = number
    })
    payments_pp_table = object({
      autoscale  = bool
      throughput = number
    })
  })
| n/a | yes | | [cosmos\_gps\_db\_params](#input\_cosmos\_gps\_db\_params) | n/a | 
object({
    kind           = string
    capabilities   = list(string)
    offer_type     = string
    server_version = string
    consistency_policy = object({
      consistency_level       = string
      max_interval_in_seconds = number
      max_staleness_prefix    = number
    })
    main_geo_location_zone_redundant = bool
    enable_free_tier                 = bool
    additional_geo_locations = list(object({
      location          = string
      failover_priority = number
      zone_redundant    = bool
    }))
    private_endpoint_enabled          = bool
    public_network_access_enabled     = bool
    is_virtual_network_filter_enabled = bool
    backup_continuous_enabled         = bool
  })
| n/a | yes | | [dns\_zone\_internal\_prefix](#input\_dns\_zone\_internal\_prefix) | The dns subdomain. | `string` | `null` | no | @@ -172,6 +185,7 @@ | [gpd\_sftp\_sa\_snet\_private\_link\_service\_network\_policies\_enabled](#input\_gpd\_sftp\_sa\_snet\_private\_link\_service\_network\_policies\_enabled) | If true, create a private link service | `bool` | `true` | no | | [gpd\_sftp\_sa\_tier\_to\_archive](#input\_gpd\_sftp\_sa\_tier\_to\_archive) | Number of days after which the blob is moved to archive | `number` | `-1` | no | | [gpd\_sftp\_sa\_tier\_to\_cool](#input\_gpd\_sftp\_sa\_tier\_to\_cool) | Number of days after which the blob is moved to cool | `number` | n/a | yes | +| [gpd\_upload\_status\_throughput](#input\_gpd\_upload\_status\_throughput) | Max container throughput (Cosmos-RU) | `number` | `1000` | no | | [ingress\_load\_balancer\_ip](#input\_ingress\_load\_balancer\_ip) | n/a | `string` | n/a | yes | | [instance](#input\_instance) | One of beta, prod01, prod02 | `string` | n/a | yes | | [location](#input\_location) | One of westeurope, northeurope | `string` | n/a | yes | @@ -182,7 +196,7 @@ | [log\_analytics\_workspace\_resource\_group\_name](#input\_log\_analytics\_workspace\_resource\_group\_name) | The name of the resource group in which the Log Analytics workspace is located in. | `string` | n/a | yes | | [monitor\_resource\_group\_name](#input\_monitor\_resource\_group\_name) | Monitor resource group name | `string` | n/a | yes | | [pgflex\_public\_metric\_alerts](#input\_pgflex\_public\_metric\_alerts) | Map of name = criteria objects | 
map(object({
    # criteria.*.aggregation to be one of [Average Count Minimum Maximum Total]
    aggregation = string
    # "Insights.Container/pods" "Insights.Container/nodes"
    metric_namespace = string
    metric_name      = string
    # criteria.0.operator to be one of [Equals NotEquals GreaterThan GreaterThanOrEqual LessThan LessThanOrEqual]
    operator  = string
    threshold = number
    # Possible values are PT1M, PT5M, PT15M, PT30M and PT1H
    frequency = string
    # Possible values are PT1M, PT5M, PT15M, PT30M, PT1H, PT6H, PT12H and P1D.
    window_size = string
    # severity: The severity of this Metric Alert. Possible values are 0, 1, 2, 3 and 4. Defaults to 3. Lower is worst
    severity = number
  }))
| 
{
  "active_connections": {
    "aggregation": "Average",
    "frequency": "PT5M",
    "metric_name": "active_connections",
    "metric_namespace": "Microsoft.DBforPostgreSQL/flexibleServers",
    "operator": "GreaterThan",
    "severity": 2,
    "threshold": 4000,
    "window_size": "PT30M"
  },
  "connections_failed": {
    "aggregation": "Total",
    "frequency": "PT5M",
    "metric_name": "connections_failed",
    "metric_namespace": "Microsoft.DBforPostgreSQL/flexibleServers",
    "operator": "GreaterThan",
    "severity": 2,
    "threshold": 10,
    "window_size": "PT30M"
  },
  "cpu_percent": {
    "aggregation": "Average",
    "frequency": "PT5M",
    "metric_name": "cpu_percent",
    "metric_namespace": "Microsoft.DBforPostgreSQL/flexibleServers",
    "operator": "GreaterThan",
    "severity": 2,
    "threshold": 80,
    "window_size": "PT30M"
  },
  "memory_percent": {
    "aggregation": "Average",
    "frequency": "PT5M",
    "metric_name": "memory_percent",
    "metric_namespace": "Microsoft.DBforPostgreSQL/flexibleServers",
    "operator": "GreaterThan",
    "severity": 2,
    "threshold": 80,
    "window_size": "PT30M"
  },
  "storage_percent": {
    "aggregation": "Average",
    "frequency": "PT5M",
    "metric_name": "storage_percent",
    "metric_namespace": "Microsoft.DBforPostgreSQL/flexibleServers",
    "operator": "GreaterThan",
    "severity": 2,
    "threshold": 80,
    "window_size": "PT30M"
  }
}
| no | -| [pgres\_flex\_params](#input\_pgres\_flex\_params) | Postgres Flexible | 
object({
    private_endpoint_enabled                         = bool
    sku_name                                         = string
    db_version                                       = string
    storage_mb                                       = string
    zone                                             = number
    backup_retention_days                            = number
    geo_redundant_backup_enabled                     = bool
    high_availability_enabled                        = bool
    standby_availability_zone                        = number
    pgbouncer_enabled                                = bool
    alerts_enabled                                   = bool
    max_connections                                  = number
    enable_private_dns_registration                  = optional(bool, false)
    enable_private_dns_registration_virtual_endpoint = optional(bool, false)
  })
| `null` | no | +| [pgres\_flex\_params](#input\_pgres\_flex\_params) | Postgres Flexible | 
object({
    private_endpoint_enabled                         = bool
    sku_name                                         = string
    db_version                                       = string
    storage_mb                                       = string
    zone                                             = number
    backup_retention_days                            = number
    geo_redundant_backup_enabled                     = bool
    high_availability_enabled                        = bool
    standby_availability_zone                        = number
    pgbouncer_enabled                                = bool
    alerts_enabled                                   = bool
    max_connections                                  = number
    enable_private_dns_registration                  = optional(bool, false)
    enable_private_dns_registration_virtual_endpoint = optional(bool, false)
    max_worker_process                               = number
    wal_level                                        = string
    shared_preoload_libraries                        = string
    public_network_access_enabled                    = bool
  })
| `null` | no | | [postgresql\_network\_rules](#input\_postgresql\_network\_rules) | Network rules restricting access to the postgresql server. | 
object({
    ip_rules                       = list(string)
    allow_access_to_azure_services = bool
  })
| 
{
  "allow_access_to_azure_services": false,
  "ip_rules": []
}
| no | | [prefix](#input\_prefix) | n/a | `string` | n/a | yes | | [reporting\_storage\_account](#input\_reporting\_storage\_account) | n/a | 
object({
    advanced_threat_protection = bool
    blob_delete_retention_days = number
    blob_versioning_enabled    = bool
    backup_enabled             = bool
    backup_retention           = optional(number, 0)
  })
| 
{
  "advanced_threat_protection": false,
  "backup_enabled": false,
  "backup_retention": 0,
  "blob_delete_retention_days": 30,
  "blob_versioning_enabled": false
}
| no | @@ -192,4 +206,4 @@ ## Outputs No outputs. - + diff --git a/src/domains/gps-secret/secret/weu-dev/noedit_secret_enc.json b/src/domains/gps-secret/secret/weu-dev/noedit_secret_enc.json index f0cb348fac..d0f67c9fbf 100644 --- a/src/domains/gps-secret/secret/weu-dev/noedit_secret_enc.json +++ b/src/domains/gps-secret/secret/weu-dev/noedit_secret_enc.json @@ -2,6 +2,7 @@ "config-cache-subscription-key": "ENC[AES256_GCM,data:p0XwVgwabL1Qo2TineWUPdMkkT89sus7oKNu8IOYN7E=,iv:hyHUqstm4TiddpBj0kimAehg9rtpfULkgL1XxN2y17I=,tag:RPfhhsPQRCFSOIrPMR8UYg==,type:str]", "cdc-logical-replication-apd-user": "ENC[AES256_GCM,data:A3+k23I+,iv:TygCKKbqy37+7f9j/V40um/ADNmuBkkpRdhY1FNNn7Y=,tag:wVZDORQ8/OSrhz53RQuKOQ==,type:str]", "cdc-logical-replication-apd-pwd": "ENC[AES256_GCM,data:ZYpNes5H0L7wEmU=,iv:U8O5Wz6lxWVPLzeO0/soqk1OcMS34qQGgXBZ8ch3XU4=,tag:cZ0rt7H+F65lmFE2L6eugQ==,type:str]", + "tokenizer-api-key": "ENC[AES256_GCM,data:q5Hzo9lITSxU5VZBCTHaZp1Pt+tJsoDAGkdG8hOj1rrE3G0mV3OI6Q==,iv:xYXTKLzP/kpH965PsfUpTePYONCmsYO2YjXbitWygPQ=,tag:4FhsG96+j4MWmovTghqKCg==,type:str]", "sops": { "kms": null, "gcp_kms": null, @@ -16,8 +17,8 @@ ], "hc_vault": null, "age": null, - "lastmodified": "2024-10-28T13:10:21Z", - "mac": "ENC[AES256_GCM,data:pevQh89zNXQlUerFdxAFidoWlXfD6UDPcq1dxegS4ZLQY44n9vXS9+A50A3kaP9nSuVaV0Ofk9cYY9sqhACBoD+el3tZl3xjeiLLvhFfYFPhmv6mX3MWWbekRfJkFqEzxT5PfMGXbWR428RV2H7Ua2NzIJTK0SIAPL0KbemsLgg=,iv:3gri06bhv8UmZ5MULAYxM9l4cSUDLHXz4GVNqL4qUFs=,tag:CnpXCR3brOU5WKYqsrUizw==,type:str]", + "lastmodified": "2024-11-12T16:45:09Z", + "mac": "ENC[AES256_GCM,data:4wJb1ErnlXKFGKsxz2J0Ec4HhFa7ZgmDPdiJBA8hyGD24kShjiXFPLhNrV11l1ebgseHEAchbZvw5x9CpFgGQto5d8jkSk66SVJL1U5q0krMWGK4Fv2U3JS2ebKjWOwg0sn7t+ypOa0f5WslhxU0UM8CyS/7zhrfqOVXEJAJplA=,iv:gyQzmiCv9bw+VXUxRTvJ1VlFgPOjBXZgERr7G64wEvc=,tag:+tU9hkA3wAkpII1/uHZ6yQ==,type:str]", "pgp": null, "unencrypted_suffix": "_unencrypted", "version": "3.9.1" diff --git a/src/domains/observability/03_gpd_ingestion_sa.tf b/src/domains/observability/03_gpd_ingestion_sa.tf new file mode 100644 index 0000000000..98455b3525 --- /dev/null +++ b/src/domains/observability/03_gpd_ingestion_sa.tf @@ -0,0 +1,36 @@ +# gpd_rg +resource "azurerm_resource_group" "gpd_ingestion_rg" { + name = "${local.project_itn}-gpd-rg" + location = var.location_itn + + tags = var.tags +} + +module "gpd_ingestion_sa" { + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account?ref=v7.18.0" + + name = replace(format("%s-gpd-ingest-sa", local.product), "-", "") + account_kind = "StorageV2" + account_tier = "Standard" + account_replication_type = var.gpd_ingestion_storage_account.account_replication_type + access_tier = "Hot" + blob_versioning_enabled = var.gpd_ingestion_storage_account.blob_versioning_enabled + resource_group_name = azurerm_resource_group.gpd_ingestion_rg.name + location = var.location_itn + advanced_threat_protection = var.gpd_ingestion_storage_account.advanced_threat_protection + allow_nested_items_to_be_public = false + public_network_access_enabled = var.gpd_ingestion_storage_account.public_network_access_enabled + enable_low_availability_alert = false + + blob_delete_retention_days = var.gpd_ingestion_storage_account.blob_delete_retention_days + + blob_change_feed_enabled = var.gpd_ingestion_storage_account.backup_enabled + blob_change_feed_retention_in_days = var.gpd_ingestion_storage_account.backup_enabled ? var.gpd_ingestion_storage_account.backup_retention + 1 : null + blob_container_delete_retention_days = var.gpd_ingestion_storage_account.backup_retention + blob_storage_policy = { + enable_immutability_policy = false + blob_restore_policy_days = var.gpd_ingestion_storage_account.backup_retention + } + + tags = var.tags +} diff --git a/src/domains/observability/99_variables.tf b/src/domains/observability/99_variables.tf index f6253b1bd0..8e96ecdb89 100644 --- a/src/domains/observability/99_variables.tf +++ b/src/domains/observability/99_variables.tf @@ -349,3 +349,26 @@ variable "cidr_subnet_observability_gpd_evh" { description = "Address prefixes evh" } +# GPD ingestion sa + +variable "gpd_ingestion_storage_account" { + type = object({ + advanced_threat_protection = bool + blob_delete_retention_days = number + blob_versioning_enabled = bool + backup_enabled = bool + backup_retention = optional(number, 0) + account_replication_type = string + public_network_access_enabled = bool + + }) + default = { + blob_versioning_enabled = false + advanced_threat_protection = false + blob_delete_retention_days = 30 + backup_enabled = false + backup_retention = 0 + account_replication_type = "LRS" // changhe to GZRS for PROD + public_network_access_enabled = true + } +} diff --git a/src/domains/observability/README.md b/src/domains/observability/README.md index 8a6cce15d1..d23ffe99e3 100644 --- a/src/domains/observability/README.md +++ b/src/domains/observability/README.md @@ -1,7 +1,7 @@ # observability - + ## Requirements | Name | Version | @@ -23,6 +23,7 @@ | [eventhub\_namespace\_observability\_gpd](#module\_eventhub\_namespace\_observability\_gpd) | git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub | v8.22.0 | | [eventhub\_observability\_configuration](#module\_eventhub\_observability\_configuration) | git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration | v8.22.0 | | [eventhub\_observability\_gpd\_configuration](#module\_eventhub\_observability\_gpd\_configuration) | git::https://github.com/pagopa/terraform-azurerm-v3.git//eventhub_configuration | v8.22.0 | +| [gpd\_ingestion\_sa](#module\_gpd\_ingestion\_sa) | git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account | v7.18.0 | | [observability\_sa](#module\_observability\_sa) | git::https://github.com/pagopa/terraform-azurerm-v3.git//storage_account | v7.18.0 | | [observability\_st\_snet](#module\_observability\_st\_snet) | git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet | v6.7.0 | @@ -93,6 +94,7 @@ | [azurerm_kusto_eventhub_data_connection.eventhub_connection_for_re_event](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/kusto_eventhub_data_connection) | resource | | [azurerm_private_endpoint.observability_storage_private_endpoint](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/private_endpoint) | resource | | [azurerm_resource_group.eventhub_observability_rg](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/resource_group) | resource | +| [azurerm_resource_group.gpd_ingestion_rg](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/resource_group) | resource | | [azurerm_resource_group.st_observability_rg](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/resource_group) | resource | | [azurerm_storage_container.blob-observability-st](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/storage_container) | resource | | [azurerm_subnet.eventhub_observability_gpd_snet](https://registry.terraform.io/providers/hashicorp/azurerm/3.53.0/docs/resources/subnet) | resource | @@ -153,6 +155,7 @@ | [eventhubs](#input\_eventhubs) | A list of event hubs to add to namespace. | 
list(object({
    name              = string
    partitions        = number
    message_retention = number
    consumers         = list(string)
    keys = list(object({
      name   = string
      listen = bool
      send   = bool
      manage = bool
    }))
  }))
| `[]` | no | | [eventhubs\_gpd](#input\_eventhubs\_gpd) | A list of event hubs to add to namespace. | 
list(object({
    name              = string
    partitions        = number
    message_retention = number
    consumers         = list(string)
    keys = list(object({
      name   = string
      listen = bool
      send   = bool
      manage = bool
    }))
  }))
| `[]` | no | | [external\_domain](#input\_external\_domain) | Domain for delegation | `string` | `null` | no | +| [gpd\_ingestion\_storage\_account](#input\_gpd\_ingestion\_storage\_account) | n/a | 
object({
    advanced_threat_protection    = bool
    blob_delete_retention_days    = number
    blob_versioning_enabled       = bool
    backup_enabled                = bool
    backup_retention              = optional(number, 0)
    account_replication_type      = string
    public_network_access_enabled = bool

  })
| 
{
  "account_replication_type": "LRS",
  "advanced_threat_protection": false,
  "backup_enabled": false,
  "backup_retention": 0,
  "blob_delete_retention_days": 30,
  "blob_versioning_enabled": false,
  "public_network_access_enabled": true
}
| no | | [instance](#input\_instance) | One of beta, prod01, prod02 | `string` | n/a | yes | | [location](#input\_location) | One of westeurope, northeurope | `string` | n/a | yes | | [location\_itn](#input\_location\_itn) | italynorth | `string` | n/a | yes | @@ -172,4 +175,4 @@ ## Outputs No outputs. - + diff --git a/src/domains/observability/env/dev/terraform.tfvars b/src/domains/observability/env/dev/terraform.tfvars index 31a750197e..6710916d23 100644 --- a/src/domains/observability/env/dev/terraform.tfvars +++ b/src/domains/observability/env/dev/terraform.tfvars @@ -261,136 +261,67 @@ ehns_metric_alerts = { }, } - - - eventhubs_gpd = [ - { - name = "test-evh" # test + { + name = "gpd-ingestion.apd.payment_option" partitions = 1 message_retention = 1 - consumers = ["test-evh"] + consumers = ["gpd-ingestion.apd.payment_option-rx-dl", ] keys = [ { - name = "test-evh" + name = "gpd-ingestion.apd.payment_option-rx-dl" listen = true + send = false + manage = false + }, + { + name = "gpd-ingestion.apd.payment_option-tx" + listen = false + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_position" + partitions = 1 + message_retention = 1 + consumers = ["gpd-ingestion.apd.payment_position-rx-dl", ] + keys = [ + { + name = "gpd-ingestion.apd.payment_position-rx-dl" + listen = true + send = false + manage = false + }, + { + name = "gpd-ingestion.apd.payment_position-tx" + listen = false + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.transfer" + partitions = 1 + message_retention = 1 + consumers = ["gpd-ingestion.apd.transfer-rx-dl", ] + keys = [ + { + name = "gpd-ingestion.apd.transfer-rx-dl" + listen = true + send = false + manage = false + }, + { + name = "gpd-ingestion.apd.transfer-tx" + listen = false send = true manage = false } ] }, - # { - # name = "connect-cluster-offsets" # debezium internal use - # partitions = 1 - # message_retention = 1 - # consumers = ["connect-cluster-offsets"] - # keys = [ - # { - # name = "connect-cluster-offsets" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "connect-cluster-status" # debezium internal use - # partitions = 1 - # message_retention = 1 - # consumers = ["connect-cluster-offsets"] - # keys = [ - # { - # name = "connect-cluster-status" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "connect-cluster-configs" # debezium internal use - # partitions = 1 - # message_retention = 1 - # consumers = ["connect-cluster-configs"] - # keys = [ - # { - # name = "connect-cluster-configs" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.payment_option" - # partitions = 1 - # message_retention = 1 - # consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] - # keys = [ - # { - # name = "gpd-ingestion.apd.payment_option-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.payment_option_metadata" - # partitions = 1 - # message_retention = 1 - # consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.payment_position" - # partitions = 1 - # message_retention = 1 - # consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.payment_position-rx-dl" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.transfer" - # partitions = 1 - # message_retention = 1 - # consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.transfer-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.transfer_metadata" - # partitions = 1 - # message_retention = 1 - # consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.transfer_metadata-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, ] diff --git a/src/domains/observability/gpd_evh_create__az.sh b/src/domains/observability/gpd_evh_create__az.sh index f200ab1088..a9d1b60ac8 100644 --- a/src/domains/observability/gpd_evh_create__az.sh +++ b/src/domains/observability/gpd_evh_create__az.sh @@ -7,38 +7,38 @@ set -e # ============================================================== -echo ">>>>>> 1" - -az eventhubs eventhub create \ --g pagopa-d-itn-observ-evh-rg \ --n "connect-cluster-offsets" \ ---namespace-name pagopa-d-itn-observ-gpd-evh \ ---cleanup-policy "Compact" \ ---status "Active" \ ---partition-count 1 \ ---retention-time 24 - -echo ">>>>>> 2" - -az eventhubs eventhub create \ --g pagopa-d-itn-observ-evh-rg \ --n "connect-cluster-status" \ ---namespace-name pagopa-d-itn-observ-gpd-evh \ ---cleanup-policy "Compact" \ ---status "Active" \ ---partition-count 1 \ ---retention-time 24 - -echo ">>>>>> 3" - -az eventhubs eventhub create \ --g pagopa-d-itn-observ-evh-rg \ --n "connect-cluster-configs" \ ---namespace-name pagopa-d-itn-observ-gpd-evh \ ---cleanup-policy "Compact" \ ---status "Active" \ ---partition-count 1 \ ---retention-time 24 +# echo ">>>>>> 1" + +# az eventhubs eventhub create \ +# -g pagopa-d-itn-observ-evh-rg \ +# -n "connect-cluster-offsets" \ +# --namespace-name pagopa-d-itn-observ-gpd-evh \ +# --cleanup-policy "Compact" \ +# --status "Active" \ +# --partition-count 1 \ +# --retention-time 24 + +# echo ">>>>>> 2" + +# az eventhubs eventhub create \ +# -g pagopa-d-itn-observ-evh-rg \ +# -n "connect-cluster-status" \ +# --namespace-name pagopa-d-itn-observ-gpd-evh \ +# --cleanup-policy "Compact" \ +# --status "Active" \ +# --partition-count 1 \ +# --retention-time 24 + +# echo ">>>>>> 3" + +# az eventhubs eventhub create \ +# -g pagopa-d-itn-observ-evh-rg \ +# -n "connect-cluster-configs" \ +# --namespace-name pagopa-d-itn-observ-gpd-evh \ +# --cleanup-policy "Compact" \ +# --status "Active" \ +# --partition-count 1 \ +# --retention-time 24 # ============================================================== # logical topics @@ -48,53 +48,53 @@ echo ">>>>>> 4" az eventhubs eventhub create \ -g pagopa-d-itn-observ-evh-rg \ --n "azcligpd-ingestion.apd.payment_option" \ +-n "cdc-raw-auto.apd.payment_option" \ --namespace-name pagopa-d-itn-observ-gpd-evh \ --cleanup-policy "Compact" \ --status "Active" \ --partition-count 1 \ --retention-time 24 -echo ">>>>>> 4" - -az eventhubs eventhub create \ --g pagopa-d-itn-observ-evh-rg \ --n "azcligpd-ingestion.apd.payment_option_metadata" \ +az eventhubs eventhub authorization-rule create \ +--resource-group pagopa-d-itn-observ-evh-rg \ --namespace-name pagopa-d-itn-observ-gpd-evh \ ---cleanup-policy "Compact" \ ---status "Active" \ ---partition-count 1 \ ---retention-time 24 +--eventhub-name cdc-raw-auto.apd.payment_option \ +--name cdc-raw-auto.apd.payment_option-rx \ +--rights Listen echo ">>>>>> 5" az eventhubs eventhub create \ -g pagopa-d-itn-observ-evh-rg \ --n "azcligpd-ingestion.apd.payment_position" \ +-n "cdc-raw-auto.apd.payment_position" \ --namespace-name pagopa-d-itn-observ-gpd-evh \ --cleanup-policy "Compact" \ --status "Active" \ --partition-count 1 \ --retention-time 24 +az eventhubs eventhub authorization-rule create \ +--resource-group pagopa-d-itn-observ-evh-rg \ +--namespace-name pagopa-d-itn-observ-gpd-evh \ +--eventhub-name cdc-raw-auto.apd.payment_position \ +--name cdc-raw-auto.apd.payment_position-rx \ +--rights Listen + echo ">>>>>> 6" az eventhubs eventhub create \ -g pagopa-d-itn-observ-evh-rg \ --n "azcligpd-ingestion.apd.transfer" \ +-n "cdc-raw-auto.apd.transfer" \ --namespace-name pagopa-d-itn-observ-gpd-evh \ --cleanup-policy "Compact" \ --status "Active" \ --partition-count 1 \ --retention-time 24 -echo ">>>>>> 7" -az eventhubs eventhub create \ --g pagopa-d-itn-observ-evh-rg \ --n "azcligpd-ingestion.apd.transfer_metadata" \ +az eventhubs eventhub authorization-rule create \ +--resource-group pagopa-d-itn-observ-evh-rg \ --namespace-name pagopa-d-itn-observ-gpd-evh \ ---cleanup-policy "Compact" \ ---status "Active" \ ---partition-count 1 \ ---retention-time 24 +--eventhub-name cdc-raw-auto.apd.transfer \ +--name cdc-raw-auto.apd.transfer-rx \ +--rights Listen diff --git a/src/domains/observability/gpd_evh_delete__az.sh b/src/domains/observability/gpd_evh_delete__az.sh index cdecbe5a06..4348f2ca03 100644 --- a/src/domains/observability/gpd_evh_delete__az.sh +++ b/src/domains/observability/gpd_evh_delete__az.sh @@ -36,63 +36,19 @@ echo ">>>>>> 4" az eventhubs eventhub delete \ -g pagopa-d-itn-observ-evh-rg \ --n "azcligpd-ingestion.apd.payment_option" \ +-n "cdc-raw-auto.apd.payment_option" \ --namespace-name pagopa-d-itn-observ-gpd-evh -echo ">>>>>> 4" - -az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ --n "azcligpd-ingestion.apd.payment_option_metadata" \ ---namespace-name pagopa-d-itn-observ-gpd-evh - - echo ">>>>>> 5" az eventhubs eventhub delete \ -g pagopa-d-itn-observ-evh-rg \ --n "azcligpd-ingestion.apd.payment_position" \ +-n "cdc-raw-auto.apd.payment_position" \ --namespace-name pagopa-d-itn-observ-gpd-evh echo ">>>>>> 6" az eventhubs eventhub delete \ -g pagopa-d-itn-observ-evh-rg \ --n "azcligpd-ingestion.apd.transfer" \ ---namespace-name pagopa-d-itn-observ-gpd-evh - -echo ">>>>>> 7" - -az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ --n "azcligpd-ingestion.apd.transfer_metadata" \ ---namespace-name pagopa-d-itn-observ-gpd-evh - -# auto-create - -echo ">>>>>> ....." - -az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ --n "azcligpd.apd.payment_option" \ ---namespace-name pagopa-d-itn-observ-gpd-evh - -az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ --n "azcligpd.apd.payment_option_metadata" \ ---namespace-name pagopa-d-itn-observ-gpd-evh - -az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ --n "azcligpd.apd.payment_position" \ ---namespace-name pagopa-d-itn-observ-gpd-evh - -az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ --n "azcligpd.apd.transfer" \ ---namespace-name pagopa-d-itn-observ-gpd-evh - -az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ --n "azcligpd.apd.transfer_metadata" \ +-n "cdc-raw-auto.apd.transfer" \ --namespace-name pagopa-d-itn-observ-gpd-evh diff --git a/src/psql/flyway_gpd.sh b/src/psql/flyway_gpd.sh index d6d52578ae..118db25097 100755 --- a/src/psql/flyway_gpd.sh +++ b/src/psql/flyway_gpd.sh @@ -103,7 +103,7 @@ printf "user [%s] pwd [%s] schema [%s]\n" "${APD_DB_USER}" "${APD_DB_PASS}" "${F # ADP USER ( GPD default ust) # https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-logical#prerequisites-for-logical-replication-and-logical-decoding -# CDC usr >>> ALTER ROLE WITH REPLICATION +# CDC usr >>> ALTER ROLE WITH REPLICATION ( ✋ creation ) docker run --rm -it --network=host -v "${WORKDIR}/migrations/${SUBSCRIPTION}/${DATABASE}":/flyway/sql \ flyway/flyway:"${FLYWAY_DOCKER_TAG}" \ -url="${FLYWAY_URL}" -user="${FLYWAY_USER}" -password="${FLYWAY_PASSWORD}" \ diff --git a/src/psql/migrations/DEV-pagoPA/apd/V2__CDC_USER.sql b/src/psql/migrations/how2_add_cdc_user.sql similarity index 100% rename from src/psql/migrations/DEV-pagoPA/apd/V2__CDC_USER.sql rename to src/psql/migrations/how2_add_cdc_user.sql From 0eb5749b43503068151ef66fa47b4cb3897f07b6 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Wed, 13 Nov 2024 14:10:39 +0100 Subject: [PATCH 32/55] fi --- src/domains/gps-common/how2_cdc_GPD.md | 75 ++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 src/domains/gps-common/how2_cdc_GPD.md diff --git a/src/domains/gps-common/how2_cdc_GPD.md b/src/domains/gps-common/how2_cdc_GPD.md new file mode 100644 index 0000000000..f5e21a97b0 --- /dev/null +++ b/src/domains/gps-common/how2_cdc_GPD.md @@ -0,0 +1,75 @@ + + + +### Step + + 1. apply https://github.com/pagopa/pagopa-infra/pull/2496 + 2. GDP network ( `+ Add 0.0.0.0 - 255.255.255.255` ) + 3. _[OPT iif not exists]_ user APD `./flyway_gpd.sh migrate -pagoPA apd apd -schemas=apd` + 4. _[OPT iif not exists]_ run [DB migration](https://github.com/pagopa/pagopa-debt-position/actions/workflows/db_migration_with_github_runner.yml) + 5. create CDC user ✋ + + ```sql + CREATE ROLE cdcapd; + ALTER ROLE cdcapd WITH INHERIT NOCREATEROLE NOCREATEDB LOGIN REPLICATION; + ALTER USER cdcapd WITH PASSWORD 'xxx'; + GRANT USAGE, CREATE ON SCHEMA apd TO cdcapd; + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA apd TO cdcapd; + GRANT SELECT, UPDATE, USAGE ON ALL SEQUENCES IN SCHEMA apd TO cdcapd; + ``` + +6. Grant pg_publication CDC + + ```sql + -- as admin + CREATE PUBLICATION dbz_publication FOR TABLE "apd"."payment_option", "apd"."payment_position", "apd"."transfer"; + -- as admin (???) + ALTER USER cdcapd CREATEDB; + -- as adp + GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA apd TO cdcapd; + -- to check + SELECT * FROM pg_publication; + SELECT * FROM pg_replication_slots; + ``` + +7. create topic via `src/domains/observability` + + ```sh + sh terraform.sh apply \ + -target=module.eventhub_namespace_observability_gpd \ + -target=module.eventhub_observability_gpd_configuration \ + -target=azurerm_eventhub_namespace_authorization_rule.cdc_connection_string \ + -target=azurerm_resource_group.gpd_ingestion_rg \ + -target=module.gpd_ingestion_sa + ``` + + + `src/domains/observability/gpd_evh_create__az.sh` for eventhub with `cleanup-policy` + +8. create `src/domains/gps-app/set_registry_secrets.sh` ( ACR pull) + +9. deploy debezium `src/domains/gps-app` + + ```sh + ./terraform.sh apply weu- \ + -target="helm_release.strimzi-kafka-operator" \ + -target="kubectl_manifest.debezium_role" \ + -target="kubectl_manifest.debezium_secrets" \ + -target="kubectl_manifest.debezoum_rbac" \ + -target="kubectl_manifest.kafka_connect" \ + -target="null_resource.wait_kafka_connect" \ + -target="kubectl_manifest.postgres_connector" \ + -target="null_resource.wait_postgres_connector" + ``` + +10. secret for gpd-mng-ingestion `src/domains/gps-common` + + ```sh + sh terraform.sh apply weu- \ + -target=azurerm_key_vault_secret.gpd_ingestion_apd_payment_option_tx_kv \ + -target=azurerm_key_vault_secret.gpd_ingestion_apd_payment_position_tx_kv \ + -target=azurerm_key_vault_secret.gpd_ingestion_apd_payment_option_transfer_tx_kv \ + -target=azurerm_key_vault_secret.cdc-raw-auto_apd_payment_option-rx_kv \ + -target=azurerm_key_vault_secret.cdc-raw-auto_apd_payment_position-rx_kv \ + -target=azurerm_key_vault_secret.cdc-raw-auto_apd_transfer-rx_kv \ + -target=azurerm_key_vault_secret.azure_web_jobs_storage_kv + ``` From 045c1026d889f9eb492ec8d8ceb38f3d5b7768f7 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Thu, 14 Nov 2024 08:25:38 +0100 Subject: [PATCH 33/55] deploy UAT --- .../gps-app/env/weu-prod/terraform.tfvars | 30 ++-- .../gps-app/env/weu-uat/terraform.tfvars | 30 ++-- src/domains/gps-app/set_registry_secrets.sh | 2 +- .../03_postgresql_gpd_TEST_2_DELETE.tf | 124 +++++++++++++ src/domains/gps-common/how2_cdc_GPD.md | 62 ++++--- .../secret/weu-uat/noedit_secret_enc.json | 5 +- .../observability/env/uat/terraform.tfvars | 169 ++++++------------ .../observability/gpd_evh_create__az.sh | 38 ++-- .../observability/gpd_evh_delete__az.sh | 25 +-- 9 files changed, 279 insertions(+), 206 deletions(-) create mode 100644 src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf diff --git a/src/domains/gps-app/env/weu-prod/terraform.tfvars b/src/domains/gps-app/env/weu-prod/terraform.tfvars index c04a5ee3fd..61c16a201c 100644 --- a/src/domains/gps-app/env/weu-prod/terraform.tfvars +++ b/src/domains/gps-app/env/weu-prod/terraform.tfvars @@ -113,19 +113,19 @@ fn_app_storage_account_info = { } ### debezium kafka conn -zookeeper_replicas = 3 +zookeeper_replicas = 3 zookeeper_request_memory = "512Mi" -zookeeper_request_cpu = 0.5 -zookeeper_limits_memory = "1024Mi" -zookeeper_limits_cpu = 1 -zookeeper_jvm_xms = "512m" -zookeeper_jvm_xmx = "1024m" -zookeeper_storage_size = "100Gi" -replicas = 3 -request_cpu = 0.5 -request_memory = "512Mi" -limits_memory = "1024Mi" -limits_cpu = 1 -postgres_db_name = "apd" -tasks_max = "1" -container_registry = "pagopadcommonacr.azurecr.io" +zookeeper_request_cpu = 0.5 +zookeeper_limits_memory = "1024Mi" +zookeeper_limits_cpu = 1 +zookeeper_jvm_xms = "512m" +zookeeper_jvm_xmx = "1024m" +zookeeper_storage_size = "100Gi" +replicas = 3 +request_cpu = 0.5 +request_memory = "512Mi" +limits_memory = "1024Mi" +limits_cpu = 1 +postgres_db_name = "apd" +tasks_max = "1" +container_registry = "pagopapcommonacr.azurecr.io" diff --git a/src/domains/gps-app/env/weu-uat/terraform.tfvars b/src/domains/gps-app/env/weu-uat/terraform.tfvars index 611b94f378..46b693d520 100644 --- a/src/domains/gps-app/env/weu-uat/terraform.tfvars +++ b/src/domains/gps-app/env/weu-uat/terraform.tfvars @@ -66,19 +66,19 @@ pgbouncer_enabled = true create_wisp_converter = true ### debezium kafka conn -zookeeper_replicas = 3 +zookeeper_replicas = 3 zookeeper_request_memory = "512Mi" -zookeeper_request_cpu = 0.5 -zookeeper_limits_memory = "1024Mi" -zookeeper_limits_cpu = 1 -zookeeper_jvm_xms = "512m" -zookeeper_jvm_xmx = "1024m" -zookeeper_storage_size = "100Gi" -replicas = 3 -request_cpu = 0.5 -request_memory = "512Mi" -limits_memory = "1024Mi" -limits_cpu = 1 -postgres_db_name = "apd" -tasks_max = "1" -container_registry = "pagopadcommonacr.azurecr.io" +zookeeper_request_cpu = 0.5 +zookeeper_limits_memory = "1024Mi" +zookeeper_limits_cpu = 1 +zookeeper_jvm_xms = "512m" +zookeeper_jvm_xmx = "1024m" +zookeeper_storage_size = "100Gi" +replicas = 1 +request_cpu = 0.5 +request_memory = "512Mi" +limits_memory = "1024Mi" +limits_cpu = 1 +postgres_db_name = "apd" +tasks_max = "1" +container_registry = "pagopaucommonacr.azurecr.io" diff --git a/src/domains/gps-app/set_registry_secrets.sh b/src/domains/gps-app/set_registry_secrets.sh index 7da581927f..5d38077a10 100644 --- a/src/domains/gps-app/set_registry_secrets.sh +++ b/src/domains/gps-app/set_registry_secrets.sh @@ -10,7 +10,7 @@ fi DOCKER_SERVER=$1 # acr url pagopacommonacr.azurecr.io DOCKER_USERNAME=$2 # acr usr pagopacommonacr DOCKER_PASSWORD=$3 # acr pwd -DOCKER_EMAIL=$4 +# DOCKER_EMAIL=$4 # Create the Docker registry secret using kubectl kubectl -n gps create secret docker-registry acr-credential \ diff --git a/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf b/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf new file mode 100644 index 0000000000..b6f6bded62 --- /dev/null +++ b/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf @@ -0,0 +1,124 @@ +## REMOVE IT after close MS issue +## Support Request: certificates-do-not-conform-to-algorithm +## ########################################################## +## ########################################################## +## ########################################################## +## ########################################################## + +# # https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-compare-single-server-flexible-server +# module "postgres_flexible_server_private_test" { # private only into UAT and PROD env +# source = "./.terraform/modules/__v3__/postgres_flexible_server" +# count = var.env_short == "u" ? 1 : 0 + +# name = format("%s-gpd-pgflex-test", local.product) + +# location = azurerm_resource_group.flex_data[0].location +# resource_group_name = azurerm_resource_group.flex_data[0].name + +# ### Network +# private_endpoint_enabled = var.pgres_flex_params.private_endpoint_enabled +# private_dns_zone_id = var.env_short != "d" ? data.azurerm_private_dns_zone.postgres[0].id : null +# delegated_subnet_id = module.postgres_flexible_snet[0].id +# public_network_access_enabled = var.pgres_flex_params.public_network_access_enabled + +# ### admin credentials +# administrator_login = data.azurerm_key_vault_secret.pgres_admin_login.value +# administrator_password = data.azurerm_key_vault_secret.pgres_admin_pwd.value + +# sku_name = var.pgres_flex_params.sku_name +# db_version = var.pgres_flex_params.db_version +# storage_mb = var.pgres_flex_params.storage_mb +# zone = var.pgres_flex_params.zone +# backup_retention_days = var.pgres_flex_params.backup_retention_days +# create_mode = null // the update of this argument triggers a replace +# geo_redundant_backup_enabled = var.pgres_flex_params.geo_redundant_backup_enabled + +# high_availability_enabled = var.pgres_flex_params.high_availability_enabled +# standby_availability_zone = var.pgres_flex_params.standby_availability_zone +# pgbouncer_enabled = var.pgres_flex_params.pgbouncer_enabled + +# diagnostic_settings_enabled = false + +# tags = var.tags + +# # alert section +# custom_metric_alerts = var.pgres_flex_params.alerts_enabled ? var.pgflex_public_metric_alerts : {} +# alerts_enabled = var.pgres_flex_params.alerts_enabled + +# alert_action = var.pgres_flex_params.alerts_enabled ? [ +# { +# action_group_id = data.azurerm_monitor_action_group.email.id +# webhook_properties = null +# }, +# { +# action_group_id = data.azurerm_monitor_action_group.slack.id +# webhook_properties = null +# }, +# { +# action_group_id = data.azurerm_monitor_action_group.opsgenie[0].id +# webhook_properties = null +# } +# ] : [] + +# private_dns_registration = var.pgres_flex_params.enable_private_dns_registration +# private_dns_zone_name = "${var.env_short}.internal.postgresql.pagopa.it" +# private_dns_zone_rg_name = data.azurerm_resource_group.rg_vnet.name +# private_dns_record_cname = "gpd-db" +# } + +# resource "azurerm_postgresql_flexible_server_database" "apd_db_flex_test" { +# count = 1 # forced + +# name = var.gpd_db_name +# server_id = module.postgres_flexible_server_private_test[0].id +# collation = "en_US.utf8" +# charset = "UTF8" +# } + +# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_connection_test" { +# count = 1 # forced + +# name = "max_connections" +# server_id = module.postgres_flexible_server_private_test[0].id +# value = var.pgres_flex_params.max_connections +# } + +# # Message : FATAL: unsupported startup parameter: extra_float_digits +# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_ignore_startup_parameters_test" { +# count = 1 # forced + +# name = "pgbouncer.ignore_startup_parameters" +# server_id = module.postgres_flexible_server_private_test[0].id +# value = "extra_float_digits" +# } + +# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_min_pool_size_test" { +# count = 1 # forced + +# name = "pgbouncer.min_pool_size" +# server_id = module.postgres_flexible_server_private_test[0].id +# value = var.env_short == "d" ? 1 : 10 +# } + +# # CDC https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-logical +# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_worker_process_test" { +# name = "max_worker_processes" +# server_id = module.postgres_flexible_server_private_test[0].id +# value = var.pgres_flex_params.max_worker_process # var.env_short == "d" ? 16 : 32 +# } + +# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_wal_level_test" { +# count = var.pgres_flex_params.wal_level != null ? 1 : 0 + +# name = "wal_level" +# server_id = module.postgres_flexible_server_private_test[0].id +# value = var.pgres_flex_params.wal_level # "logical", ... +# } + +# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_shared_preoload_libraries_test" { +# count = var.pgres_flex_params.wal_level != null ? 1 : 0 + +# name = "shared_preload_libraries" +# server_id = module.postgres_flexible_server_private_test[0].id +# value = var.pgres_flex_params.shared_preoload_libraries # "pg_failover_slots" +# } diff --git a/src/domains/gps-common/how2_cdc_GPD.md b/src/domains/gps-common/how2_cdc_GPD.md index f5e21a97b0..d81372d1ec 100644 --- a/src/domains/gps-common/how2_cdc_GPD.md +++ b/src/domains/gps-common/how2_cdc_GPD.md @@ -3,22 +3,44 @@ ### Step - 1. apply https://github.com/pagopa/pagopa-infra/pull/2496 - 2. GDP network ( `+ Add 0.0.0.0 - 255.255.255.255` ) - 3. _[OPT iif not exists]_ user APD `./flyway_gpd.sh migrate -pagoPA apd apd -schemas=apd` - 4. _[OPT iif not exists]_ run [DB migration](https://github.com/pagopa/pagopa-debt-position/actions/workflows/db_migration_with_github_runner.yml) - 5. create CDC user ✋ + +1. create topic via `src/domains/observability` + + ```sh + sh terraform.sh apply \ + -target=module.eventhub_namespace_observability_gpd \ + -target=module.eventhub_observability_gpd_configuration \ + -target=azurerm_eventhub_namespace_authorization_rule.cdc_connection_string \ + -target=azurerm_resource_group.gpd_ingestion_rg \ + -target=module.gpd_ingestion_sa + ``` + + + `src/domains/observability/gpd_evh_create__az.sh` for eventhub with `cleanup-policy` + +1. apply https://github.com/pagopa/pagopa-infra/pull/2496 ( create/config GDP db + common secrets) + NOTE : GDP network ( `+ Add 0.0.0.0 - 255.255.255.255` ) ( _only dev_ ) + +1. _[OPT iif not exists]_ user APD `./flyway_gpd.sh migrate -pagoPA apd apd -schemas=apd` + +1. _[OPT iif not exists]_ run [DB migration](https://github.com/pagopa/pagopa-debt-position/actions/workflows/db_migration_with_github_runner.yml) - ```sql +1. apply `tokenizer-api-key` secret into gpd-secrets + + +1. create CDC user ✋ + + ```sql + -- as admin CREATE ROLE cdcapd; ALTER ROLE cdcapd WITH INHERIT NOCREATEROLE NOCREATEDB LOGIN REPLICATION; ALTER USER cdcapd WITH PASSWORD 'xxx'; GRANT USAGE, CREATE ON SCHEMA apd TO cdcapd; + -- as apd GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA apd TO cdcapd; GRANT SELECT, UPDATE, USAGE ON ALL SEQUENCES IN SCHEMA apd TO cdcapd; ``` -6. Grant pg_publication CDC +1. Grant `pg_publication` CDC ```sql -- as admin @@ -32,22 +54,14 @@ SELECT * FROM pg_replication_slots; ``` -7. create topic via `src/domains/observability` - - ```sh - sh terraform.sh apply \ - -target=module.eventhub_namespace_observability_gpd \ - -target=module.eventhub_observability_gpd_configuration \ - -target=azurerm_eventhub_namespace_authorization_rule.cdc_connection_string \ - -target=azurerm_resource_group.gpd_ingestion_rg \ - -target=module.gpd_ingestion_sa +1. create `src/domains/gps-app/set_registry_secrets.sh` ( ACR pull ) + ``` + kubectl config get-contexts + kubectl config current-context + kubectl config use-context ``` - + `src/domains/observability/gpd_evh_create__az.sh` for eventhub with `cleanup-policy` - -8. create `src/domains/gps-app/set_registry_secrets.sh` ( ACR pull) - -9. deploy debezium `src/domains/gps-app` +1. deploy debezium `src/domains/gps-app` ```sh ./terraform.sh apply weu- \ @@ -59,12 +73,12 @@ -target="null_resource.wait_kafka_connect" \ -target="kubectl_manifest.postgres_connector" \ -target="null_resource.wait_postgres_connector" - ``` + ``` -10. secret for gpd-mng-ingestion `src/domains/gps-common` +1. secret for gpd-mng-ingestion `src/domains/gps-common` ```sh - sh terraform.sh apply weu- \ + sh terraform.sh apply weu- \ -target=azurerm_key_vault_secret.gpd_ingestion_apd_payment_option_tx_kv \ -target=azurerm_key_vault_secret.gpd_ingestion_apd_payment_position_tx_kv \ -target=azurerm_key_vault_secret.gpd_ingestion_apd_payment_option_transfer_tx_kv \ diff --git a/src/domains/gps-secret/secret/weu-uat/noedit_secret_enc.json b/src/domains/gps-secret/secret/weu-uat/noedit_secret_enc.json index 28b1d982f2..a4d666ca75 100644 --- a/src/domains/gps-secret/secret/weu-uat/noedit_secret_enc.json +++ b/src/domains/gps-secret/secret/weu-uat/noedit_secret_enc.json @@ -2,6 +2,7 @@ "config-cache-subscription-key": "ENC[AES256_GCM,data:ofxzb+O9YAM/mAeDwYSDVsYAkDauFNvhtcfjRAkNgRc=,iv:Mfb1u4GxtlHuB+pj9UJT8psqasjgR620VP+Mm/aSyMA=,tag:Bu9YXfb4VWM5PYRWZyDr9Q==,type:str]", "cdc-logical-replication-apd-user": "ENC[AES256_GCM,data:evSo198e,iv:kx+k5jooyprJye+mp4PH3ps+rc4WiG6F2f7LqJSIAa0=,tag:Va2WC742cqG3H3PoxydSpA==,type:str]", "cdc-logical-replication-apd-pwd": "ENC[AES256_GCM,data:xCFisM6ZVAmqn54BEPA=,iv:qTWn6zzyAzT1L5LIy7xML8QLCaskwY6ZF7FCBGJXruk=,tag:lgLrXq0XjIujfU4wITKGfg==,type:str]", + "tokenizer-api-key": "ENC[AES256_GCM,data:qfhqVH/zU9R2T+vbRdMwIWGagxao2Yui532H34U3o6jFHcHQK/JJ2w==,iv:3JchD3+KmpYreDeMv4pKYqkxzADU43r6YdwoYvNvqb4=,tag:K1OFJ/qbtiffJNAEvw5nJg==,type:str]", "sops": { "kms": null, "gcp_kms": null, @@ -16,8 +17,8 @@ ], "hc_vault": null, "age": null, - "lastmodified": "2024-10-28T13:12:13Z", - "mac": "ENC[AES256_GCM,data:u+/QWU97DMudEX/SlM2bSPlT0ozDYd6zueDFS6Ipn30HdHEMusmBmwft1p76pyu78v9/KndP1wcGq+bQangCfAbmFI6H/95Z0RNCXJqQi3RfojnBSuIqMHGIggjs3G4x/5ot8tKGCocNt4G0/M3NtkNFac3uozQTSfPjo2L66y4=,iv:B8Kyr6qYFHmgZl1RIZlI4TTJGjjBYI9arypa9X8qxwE=,tag:SuJ80liprzz/eQcq+CgZyw==,type:str]", + "lastmodified": "2024-11-13T14:19:26Z", + "mac": "ENC[AES256_GCM,data:f74/TEz+hy2Dyyl5pYGt6En8eedfo7cS1sdDhdfv0scmZ7uS5/WHxtKSEUHpVFX/Ye98CK/A6FVCOpwmrEHF/XVBrDxWbKoWDVGj7S5F8iPVH0xUcWitJdCxQU5FdDZVYLvU7uDomk/cPspWeQ9F8caAZYNzhBukSCviIQpz1+4=,iv:CVKP2mCUY43SoDsou7zqtngsfOBUtwXKGlfy8TeMqAQ=,tag:lWId9/IFzXrRw6jkHqUTFQ==,type:str]", "pgp": null, "unencrypted_suffix": "_unencrypted", "version": "3.9.1" diff --git a/src/domains/observability/env/uat/terraform.tfvars b/src/domains/observability/env/uat/terraform.tfvars index 9a043030d5..d97fe4a82d 100644 --- a/src/domains/observability/env/uat/terraform.tfvars +++ b/src/domains/observability/env/uat/terraform.tfvars @@ -281,136 +281,67 @@ ehns_metric_alerts = { }, } - - - eventhubs_gpd = [ - { - name = "test-evh" # test + { + name = "gpd-ingestion.apd.payment_option" partitions = 1 message_retention = 1 - consumers = ["test-evh"] + consumers = ["gpd-ingestion.apd.payment_option-rx-dl", ] keys = [ { - name = "test-evh" + name = "gpd-ingestion.apd.payment_option-rx-dl" listen = true + send = false + manage = false + }, + { + name = "gpd-ingestion.apd.payment_option-tx" + listen = false + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_position" + partitions = 1 + message_retention = 1 + consumers = ["gpd-ingestion.apd.payment_position-rx-dl", ] + keys = [ + { + name = "gpd-ingestion.apd.payment_position-rx-dl" + listen = true + send = false + manage = false + }, + { + name = "gpd-ingestion.apd.payment_position-tx" + listen = false + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.transfer" + partitions = 1 + message_retention = 1 + consumers = ["gpd-ingestion.apd.transfer-rx-dl", ] + keys = [ + { + name = "gpd-ingestion.apd.transfer-rx-dl" + listen = true + send = false + manage = false + }, + { + name = "gpd-ingestion.apd.transfer-tx" + listen = false send = true manage = false } ] }, - # { - # name = "connect-cluster-offsets" # debezium internal use - # partitions = 1 - # message_retention = 1 - # consumers = ["connect-cluster-offsets"] - # keys = [ - # { - # name = "connect-cluster-offsets" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "connect-cluster-status" # debezium internal use - # partitions = 1 - # message_retention = 1 - # consumers = ["connect-cluster-offsets"] - # keys = [ - # { - # name = "connect-cluster-status" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "connect-cluster-configs" # debezium internal use - # partitions = 1 - # message_retention = 1 - # consumers = ["connect-cluster-configs"] - # keys = [ - # { - # name = "connect-cluster-configs" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.payment_option" - # partitions = 1 - # message_retention = 1 - # consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] - # keys = [ - # { - # name = "gpd-ingestion.apd.payment_option-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.payment_option_metadata" - # partitions = 1 - # message_retention = 1 - # consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.payment_position" - # partitions = 1 - # message_retention = 1 - # consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.payment_position-rx-dl" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.transfer" - # partitions = 1 - # message_retention = 1 - # consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.transfer-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.transfer_metadata" - # partitions = 1 - # message_retention = 1 - # consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.transfer_metadata-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, ] diff --git a/src/domains/observability/gpd_evh_create__az.sh b/src/domains/observability/gpd_evh_create__az.sh index a9d1b60ac8..992f38b6cc 100644 --- a/src/domains/observability/gpd_evh_create__az.sh +++ b/src/domains/observability/gpd_evh_create__az.sh @@ -10,9 +10,9 @@ set -e # echo ">>>>>> 1" # az eventhubs eventhub create \ -# -g pagopa-d-itn-observ-evh-rg \ +# -g pagopa-$env-itn-observ-evh-rg \ # -n "connect-cluster-offsets" \ -# --namespace-name pagopa-d-itn-observ-gpd-evh \ +# --namespace-name pagopa-$env-itn-observ-gpd-evh \ # --cleanup-policy "Compact" \ # --status "Active" \ # --partition-count 1 \ @@ -21,9 +21,9 @@ set -e # echo ">>>>>> 2" # az eventhubs eventhub create \ -# -g pagopa-d-itn-observ-evh-rg \ +# -g pagopa-$env-itn-observ-evh-rg \ # -n "connect-cluster-status" \ -# --namespace-name pagopa-d-itn-observ-gpd-evh \ +# --namespace-name pagopa-$env-itn-observ-gpd-evh \ # --cleanup-policy "Compact" \ # --status "Active" \ # --partition-count 1 \ @@ -32,9 +32,9 @@ set -e # echo ">>>>>> 3" # az eventhubs eventhub create \ -# -g pagopa-d-itn-observ-evh-rg \ +# -g pagopa-$env-itn-observ-evh-rg \ # -n "connect-cluster-configs" \ -# --namespace-name pagopa-d-itn-observ-gpd-evh \ +# --namespace-name pagopa-$env-itn-observ-gpd-evh \ # --cleanup-policy "Compact" \ # --status "Active" \ # --partition-count 1 \ @@ -44,20 +44,22 @@ set -e # logical topics # ============================================================== +env=$1 + echo ">>>>>> 4" az eventhubs eventhub create \ --g pagopa-d-itn-observ-evh-rg \ +-g pagopa-$env-itn-observ-evh-rg \ -n "cdc-raw-auto.apd.payment_option" \ ---namespace-name pagopa-d-itn-observ-gpd-evh \ +--namespace-name pagopa-$env-itn-observ-gpd-evh \ --cleanup-policy "Compact" \ --status "Active" \ --partition-count 1 \ --retention-time 24 az eventhubs eventhub authorization-rule create \ ---resource-group pagopa-d-itn-observ-evh-rg \ ---namespace-name pagopa-d-itn-observ-gpd-evh \ +--resource-group pagopa-$env-itn-observ-evh-rg \ +--namespace-name pagopa-$env-itn-observ-gpd-evh \ --eventhub-name cdc-raw-auto.apd.payment_option \ --name cdc-raw-auto.apd.payment_option-rx \ --rights Listen @@ -65,17 +67,17 @@ az eventhubs eventhub authorization-rule create \ echo ">>>>>> 5" az eventhubs eventhub create \ --g pagopa-d-itn-observ-evh-rg \ +-g pagopa-$env-itn-observ-evh-rg \ -n "cdc-raw-auto.apd.payment_position" \ ---namespace-name pagopa-d-itn-observ-gpd-evh \ +--namespace-name pagopa-$env-itn-observ-gpd-evh \ --cleanup-policy "Compact" \ --status "Active" \ --partition-count 1 \ --retention-time 24 az eventhubs eventhub authorization-rule create \ ---resource-group pagopa-d-itn-observ-evh-rg \ ---namespace-name pagopa-d-itn-observ-gpd-evh \ +--resource-group pagopa-$env-itn-observ-evh-rg \ +--namespace-name pagopa-$env-itn-observ-gpd-evh \ --eventhub-name cdc-raw-auto.apd.payment_position \ --name cdc-raw-auto.apd.payment_position-rx \ --rights Listen @@ -83,9 +85,9 @@ az eventhubs eventhub authorization-rule create \ echo ">>>>>> 6" az eventhubs eventhub create \ --g pagopa-d-itn-observ-evh-rg \ +-g pagopa-$env-itn-observ-evh-rg \ -n "cdc-raw-auto.apd.transfer" \ ---namespace-name pagopa-d-itn-observ-gpd-evh \ +--namespace-name pagopa-$env-itn-observ-gpd-evh \ --cleanup-policy "Compact" \ --status "Active" \ --partition-count 1 \ @@ -93,8 +95,8 @@ az eventhubs eventhub create \ az eventhubs eventhub authorization-rule create \ ---resource-group pagopa-d-itn-observ-evh-rg \ ---namespace-name pagopa-d-itn-observ-gpd-evh \ +--resource-group pagopa-$env-itn-observ-evh-rg \ +--namespace-name pagopa-$env-itn-observ-gpd-evh \ --eventhub-name cdc-raw-auto.apd.transfer \ --name cdc-raw-auto.apd.transfer-rx \ --rights Listen diff --git a/src/domains/observability/gpd_evh_delete__az.sh b/src/domains/observability/gpd_evh_delete__az.sh index 4348f2ca03..dd059f963b 100644 --- a/src/domains/observability/gpd_evh_delete__az.sh +++ b/src/domains/observability/gpd_evh_delete__az.sh @@ -6,27 +6,28 @@ set -e # config topics # ============================================================== +env=$1 echo ">>>>>> 1" az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ +-g pagopa-$env-itn-observ-evh-rg \ -n "connect-cluster-offsets" \ ---namespace-name pagopa-d-itn-observ-gpd-evh +--namespace-name pagopa-$env-itn-observ-gpd-evh echo ">>>>>> 2" az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ +-g pagopa-$env-itn-observ-evh-rg \ -n "connect-cluster-status" \ ---namespace-name pagopa-d-itn-observ-gpd-evh +--namespace-name pagopa-$env-itn-observ-gpd-evh echo ">>>>>> 3" az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ +-g pagopa-$env-itn-observ-evh-rg \ -n "connect-cluster-configs" \ ---namespace-name pagopa-d-itn-observ-gpd-evh +--namespace-name pagopa-$env-itn-observ-gpd-evh # ============================================================== # logical topics @@ -35,20 +36,20 @@ az eventhubs eventhub delete \ echo ">>>>>> 4" az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ +-g pagopa-$env-itn-observ-evh-rg \ -n "cdc-raw-auto.apd.payment_option" \ ---namespace-name pagopa-d-itn-observ-gpd-evh +--namespace-name pagopa-$env-itn-observ-gpd-evh echo ">>>>>> 5" az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ +-g pagopa-$env-itn-observ-evh-rg \ -n "cdc-raw-auto.apd.payment_position" \ ---namespace-name pagopa-d-itn-observ-gpd-evh +--namespace-name pagopa-$env-itn-observ-gpd-evh echo ">>>>>> 6" az eventhubs eventhub delete \ --g pagopa-d-itn-observ-evh-rg \ +-g pagopa-$env-itn-observ-evh-rg \ -n "cdc-raw-auto.apd.transfer" \ ---namespace-name pagopa-d-itn-observ-gpd-evh +--namespace-name pagopa-$env-itn-observ-gpd-evh From d4605e3ac6ddda0afda617c21d6770067f25b809 Mon Sep 17 00:00:00 2001 From: acialini Date: Thu, 14 Nov 2024 12:27:48 +0100 Subject: [PATCH 34/55] [PPANTT-171] feat: introduced ingestion manager to the status page --- src/domains/shared-app/04_apim_statuspage.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/src/domains/shared-app/04_apim_statuspage.tf b/src/domains/shared-app/04_apim_statuspage.tf index 6a495eaf61..48baad3f1f 100644 --- a/src/domains/shared-app/04_apim_statuspage.tf +++ b/src/domains/shared-app/04_apim_statuspage.tf @@ -150,6 +150,7 @@ module "apim_api_statuspage_api_v1" { "gpdreportinganalysis" = format("%s/", data.azurerm_function_app.reporting_analysis.default_hostname) "gpdreportingbatch" = format("%s/api/", data.azurerm_function_app.reporting_batch.default_hostname) "gpdreportingservice" = format("%s/api/", data.azurerm_function_app.reporting_service.default_hostname) + "gpdingestionmanager" = format("%s/pagopa-gpd-ingestion-manager", format(local.aks_path, "gps")) "gps" = format("%s/pagopa-spontaneous-payments-service", format(local.aks_path, "gps")) "gpsdonation" = format("%s/pagopa-gps-donation-service", format(local.aks_path, "gps")) "mockec" = var.env_short != "p" ? format("%s/", data.azurerm_linux_function_app.mockec[0].default_hostname) : "NA" From f532fe806b91840cffe6ec74ac4c96aba3b904b0 Mon Sep 17 00:00:00 2001 From: acialini Date: Fri, 15 Nov 2024 09:57:12 +0100 Subject: [PATCH 35/55] [PPANTT-186] feat: introduced alerts for ingestion manager functions --- src/domains/gps-app/00_alert_gpd_ingestion.tf | 225 ++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 src/domains/gps-app/00_alert_gpd_ingestion.tf diff --git a/src/domains/gps-app/00_alert_gpd_ingestion.tf b/src/domains/gps-app/00_alert_gpd_ingestion.tf new file mode 100644 index 0000000000..8f609f3e05 --- /dev/null +++ b/src/domains/gps-app/00_alert_gpd_ingestion.tf @@ -0,0 +1,225 @@ +locals { + + fn_name_for_alerts_exceptions = var.env_short == "d" ? [] : [ + { + id: "paymentoptionprocessor" + name : "PaymentOptionProcessor" + }, + { + id: "paymentpositionprocessor" + name : "PaymentPositionProcessor" + }, + { + id: "transferprocessor" + name : "TransferProcessor" + } + ] + + + action_groups_default = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id] + + # ENABLE PROD afert deploy + action_groups = var.env_short == "p" ? concat(local.action_groups_default, [data.azurerm_monitor_action_group.opsgenie[0].id]) : local.action_groups_default + # action_groups = local.action_groups_default +} + +resource "azurerm_monitor_scheduled_query_rules_alert" "gpd-ingestion-manager-availability" { + for_each = { for c in local.fn_name_for_alerts_exceptions : c.name => c } + resource_group_name = "dashboards" + name = "pagopa-${var.env_short}-gpd-ingestion-manager-availability-${each.value.id}" + location = var.location + + action { + action_group = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id, data.azurerm_monitor_action_group.opsgenie[0].id] + # action_group = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id] + email_subject = "gpd-ingestion-manager-availability ${each.value.name}" + custom_webhook_payload = "{}" + } + data_source_id = data.azurerm_application_insights.application_insights.id + description = "Availability gpd-ingestion ${each.value.name}" + enabled = true + query = format(<<-QUERY +let threshold = 0.99; +union traces, exceptions +| where cloud_RoleName == "pagopa-gpd-ingestion-manager" +| where operation_Name == "%s" +//| summarize count() by operation_Name, itemType +| summarize + Total=count(), + Success=count(itemType == "trace") + by bin(timestamp, 10m) +| extend availability=toreal(Success) / Total +//| render timechart +| where availability < threshold + QUERY + , each.value.name) + severity = 1 + frequency = 5 + time_window = 5 + trigger { + operator = "GreaterThanOrEqual" + threshold = 2 + } +} + +resource "azurerm_monitor_scheduled_query_rules_alert" "gpd-ingestion-manager-error-json" { + for_each = { for c in local.fn_name_for_alerts_exceptions : c.name => c } + resource_group_name = "dashboards" + name = "pagopa-${var.env_short}-gpd-ingestion-manager-error-json-${each.value.id}" + location = var.location + + action { + action_group = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id, data.azurerm_monitor_action_group.opsgenie[0].id] + # action_group = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id] + email_subject = "gpd-ingestion-manager-error-json ${each.value.name}" + custom_webhook_payload = "{}" + } + data_source_id = data.azurerm_application_insights.application_insights.id + description = "Error on JsonProcessing gpd-ingestion ${each.value.name}" + enabled = true + query = format(<<-QUERY + traces + | where cloud_RoleName == "%s" + | order by timestamp desc + | where message contains "function error JsonProcessingException" + QUERY + , "pagopa-gpd-ingestion-manager" + ) + severity = 2 // Sev 2 Warning + frequency = 15 + time_window = 15 + trigger { + operator = "GreaterThanOrEqual" + threshold = 20 + } +} + +resource "azurerm_monitor_scheduled_query_rules_alert" "gpd-ingestion-manager-error-generic" { + for_each = { for c in local.fn_name_for_alerts_exceptions : c.name => c } + resource_group_name = "dashboards" + name = "pagopa-${var.env_short}-gpd-ingestion-manager-error-generic-${each.value.id}" + location = var.location + + action { + action_group = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id, data.azurerm_monitor_action_group.opsgenie[0].id] + # action_group = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id] + email_subject = "gpd-ingestion-manager-error-generic ${each.value.name}" + custom_webhook_payload = "{}" + } + data_source_id = data.azurerm_application_insights.application_insights.id + description = "Error on GenericError gpd-ingestion ${each.value.name}" + enabled = true + query = format(<<-QUERY + traces + | where cloud_RoleName == "%s" + | order by timestamp desc + | where message contains "function error Generic exception at" + QUERY + , "pagopa-gpd-ingestion-manager" + ) + severity = 2 // Sev 2 Warning + frequency = 15 + time_window = 15 + trigger { + operator = "GreaterThanOrEqual" + threshold = 20 + } +} + +resource "azurerm_monitor_scheduled_query_rules_alert" "gpd-ingestion-manager-error-pdv-tokenizer" { + for_each = { for c in local.fn_name_for_alerts_exceptions : c.name => c } + resource_group_name = "dashboards" + name = "pagopa-${var.env_short}-gpd-ingestion-manager-error-pdv-tokenizer-${each.value.id}" + location = var.location + + action { + action_group = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id, data.azurerm_monitor_action_group.opsgenie[0].id] + # action_group = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id] + email_subject = "gpd-ingestion-manager-error-pdv-tokenizer ${each.value.name}" + custom_webhook_payload = "{}" + } + data_source_id = data.azurerm_application_insights.application_insights.id + description = "Error on PDVTokenizerException gpd-ingestion ${each.value.name}" + enabled = true + query = format(<<-QUERY + traces + | where cloud_RoleName == "%s" + | order by timestamp desc + | where message contains "function error PDVTokenizerException exception at" + QUERY + , "pagopa-gpd-ingestion-manager" + ) + severity = 2 // Sev 2 Warning + frequency = 15 + time_window = 15 + trigger { + operator = "GreaterThanOrEqual" + threshold = 20 + } +} + +resource "azurerm_monitor_scheduled_query_rules_alert" "gpd-ingestion-manager-error-unexpected-pdv-tokenizer" { + for_each = { for c in local.fn_name_for_alerts_exceptions : c.name => c } + resource_group_name = "dashboards" + name = "pagopa-${var.env_short}-gpd-ingestion-manager-error-unexpected-pdv-tokenizer-${each.value.id}" + location = var.location + + action { + action_group = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id, data.azurerm_monitor_action_group.opsgenie[0].id] + # action_group = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id] + email_subject = "gpd-ingestion-manager-error-unexpected-pdv-tokenizer ${each.value.name}" + custom_webhook_payload = "{}" + } + data_source_id = data.azurerm_application_insights.application_insights.id + description = "Error on PDVTokenizerUnexpectedException gpd-ingestion ${each.value.name}" + enabled = true + query = format(<<-QUERY + traces + | where cloud_RoleName == "%s" + | order by timestamp desc + | where message contains "function error PDVTokenizerUnexpectedException exception at" + QUERY + , "pagopa-gpd-ingestion-manager" + ) + severity = 2 // Sev 2 Warning + frequency = 15 + time_window = 15 + trigger { + operator = "GreaterThanOrEqual" + threshold = 20 + } +} + +resource "azurerm_monitor_scheduled_query_rules_alert" "gpd-ingestion-manager-error-alert" { + for_each = { for c in local.fn_name_for_alerts_exceptions : c.name => c } + + resource_group_name = "dashboards" + name = "pagopa-${var.env_short}-gpd-ingestion-manager-error-alert" + location = var.location + + action { + # action_group = [data.azurerm_monitor_action_group.email.id, data.azurerm_monitor_action_group.slack.id] + action_group = local.action_groups + email_subject = "Unexpected error while managing gpd ingestion events" + custom_webhook_payload = "{}" + } + data_source_id = data.azurerm_application_insights.application_insights.id + description = "Binding exception on gpd-ingestion-manager" + enabled = true + query = format(<<-QUERY + exceptions + | where cloud_RoleName == "%s" + | where outerMessage contains "Exception while executing function: Functions.${each.value.name}" + | order by timestamp desc + QUERY + , "pagopa-gpd-ingestion-manager" # from HELM's parameter WEBSITE_SITE_NAME + ) + severity = 2 // Sev 2 Warning + frequency = 15 + time_window = 15 + trigger { + operator = "GreaterThanOrEqual" + threshold = 20 + } + +} From a8a6c3e70ded703a3730bdadf627595b2bb72b91 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Fri, 15 Nov 2024 15:25:12 +0100 Subject: [PATCH 36/55] fix --- .../03_postgresql_gpd_TEST_2_DELETE.tf | 250 +++++++++--------- .../gps-common/env/weu-prod/terraform.tfvars | 2 +- src/domains/observability/05_app_forwarder.tf | 6 +- .../observability/env/prod/terraform.tfvars | 173 ++++-------- 4 files changed, 183 insertions(+), 248 deletions(-) diff --git a/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf b/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf index b6f6bded62..090f78f851 100644 --- a/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf +++ b/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf @@ -1,124 +1,126 @@ -## REMOVE IT after close MS issue -## Support Request: certificates-do-not-conform-to-algorithm -## ########################################################## -## ########################################################## -## ########################################################## -## ########################################################## - -# # https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-compare-single-server-flexible-server -# module "postgres_flexible_server_private_test" { # private only into UAT and PROD env -# source = "./.terraform/modules/__v3__/postgres_flexible_server" -# count = var.env_short == "u" ? 1 : 0 - -# name = format("%s-gpd-pgflex-test", local.product) - -# location = azurerm_resource_group.flex_data[0].location -# resource_group_name = azurerm_resource_group.flex_data[0].name - -# ### Network -# private_endpoint_enabled = var.pgres_flex_params.private_endpoint_enabled -# private_dns_zone_id = var.env_short != "d" ? data.azurerm_private_dns_zone.postgres[0].id : null -# delegated_subnet_id = module.postgres_flexible_snet[0].id -# public_network_access_enabled = var.pgres_flex_params.public_network_access_enabled - -# ### admin credentials -# administrator_login = data.azurerm_key_vault_secret.pgres_admin_login.value -# administrator_password = data.azurerm_key_vault_secret.pgres_admin_pwd.value - -# sku_name = var.pgres_flex_params.sku_name -# db_version = var.pgres_flex_params.db_version -# storage_mb = var.pgres_flex_params.storage_mb -# zone = var.pgres_flex_params.zone -# backup_retention_days = var.pgres_flex_params.backup_retention_days -# create_mode = null // the update of this argument triggers a replace -# geo_redundant_backup_enabled = var.pgres_flex_params.geo_redundant_backup_enabled - -# high_availability_enabled = var.pgres_flex_params.high_availability_enabled -# standby_availability_zone = var.pgres_flex_params.standby_availability_zone -# pgbouncer_enabled = var.pgres_flex_params.pgbouncer_enabled - -# diagnostic_settings_enabled = false - -# tags = var.tags - -# # alert section -# custom_metric_alerts = var.pgres_flex_params.alerts_enabled ? var.pgflex_public_metric_alerts : {} -# alerts_enabled = var.pgres_flex_params.alerts_enabled - -# alert_action = var.pgres_flex_params.alerts_enabled ? [ -# { -# action_group_id = data.azurerm_monitor_action_group.email.id -# webhook_properties = null -# }, -# { -# action_group_id = data.azurerm_monitor_action_group.slack.id -# webhook_properties = null -# }, -# { -# action_group_id = data.azurerm_monitor_action_group.opsgenie[0].id -# webhook_properties = null -# } -# ] : [] - -# private_dns_registration = var.pgres_flex_params.enable_private_dns_registration -# private_dns_zone_name = "${var.env_short}.internal.postgresql.pagopa.it" -# private_dns_zone_rg_name = data.azurerm_resource_group.rg_vnet.name -# private_dns_record_cname = "gpd-db" -# } - -# resource "azurerm_postgresql_flexible_server_database" "apd_db_flex_test" { -# count = 1 # forced - -# name = var.gpd_db_name -# server_id = module.postgres_flexible_server_private_test[0].id -# collation = "en_US.utf8" -# charset = "UTF8" -# } - -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_connection_test" { -# count = 1 # forced - -# name = "max_connections" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = var.pgres_flex_params.max_connections -# } - -# # Message : FATAL: unsupported startup parameter: extra_float_digits -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_ignore_startup_parameters_test" { -# count = 1 # forced - -# name = "pgbouncer.ignore_startup_parameters" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = "extra_float_digits" -# } - -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_min_pool_size_test" { -# count = 1 # forced - -# name = "pgbouncer.min_pool_size" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = var.env_short == "d" ? 1 : 10 -# } - -# # CDC https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-logical -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_worker_process_test" { -# name = "max_worker_processes" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = var.pgres_flex_params.max_worker_process # var.env_short == "d" ? 16 : 32 -# } - -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_wal_level_test" { -# count = var.pgres_flex_params.wal_level != null ? 1 : 0 - -# name = "wal_level" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = var.pgres_flex_params.wal_level # "logical", ... -# } - -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_shared_preoload_libraries_test" { -# count = var.pgres_flex_params.wal_level != null ? 1 : 0 - -# name = "shared_preload_libraries" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = var.pgres_flex_params.shared_preoload_libraries # "pg_failover_slots" -# } +# REMOVE IT after close MS issue +# Support Request: certificates-do-not-conform-to-algorithm +# ########################################################## +# ########################################################## +# ########################################################## +# ########################################################## + +# https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-compare-single-server-flexible-server +module "postgres_flexible_server_private_test" { # private only into UAT and PROD env + source = "./.terraform/modules/__v3__/postgres_flexible_server" + count = var.env_short == "u" ? 1 : 0 + + name = format("%s-gpd-pgflex-test", local.product) + + location = azurerm_resource_group.flex_data[0].location + resource_group_name = azurerm_resource_group.flex_data[0].name + + ### Network + private_endpoint_enabled = var.pgres_flex_params.private_endpoint_enabled + private_dns_zone_id = var.env_short != "d" ? data.azurerm_private_dns_zone.postgres[0].id : null + delegated_subnet_id = module.postgres_flexible_snet[0].id + public_network_access_enabled = var.pgres_flex_params.public_network_access_enabled + + ### admin credentials + administrator_login = data.azurerm_key_vault_secret.pgres_admin_login.value + administrator_password = data.azurerm_key_vault_secret.pgres_admin_pwd.value + + sku_name = var.pgres_flex_params.sku_name + db_version = var.pgres_flex_params.db_version + storage_mb = var.pgres_flex_params.storage_mb + zone = var.pgres_flex_params.zone + backup_retention_days = var.pgres_flex_params.backup_retention_days + create_mode = null // the update of this argument triggers a replace + geo_redundant_backup_enabled = var.pgres_flex_params.geo_redundant_backup_enabled + + high_availability_enabled = var.pgres_flex_params.high_availability_enabled + standby_availability_zone = var.pgres_flex_params.standby_availability_zone + pgbouncer_enabled = var.pgres_flex_params.pgbouncer_enabled + + diagnostic_settings_enabled = false + + tags = var.tags + + # alert section + custom_metric_alerts = var.pgres_flex_params.alerts_enabled ? var.pgflex_public_metric_alerts : {} + alerts_enabled = var.pgres_flex_params.alerts_enabled + + alert_action = var.pgres_flex_params.alerts_enabled ? [ + { + action_group_id = data.azurerm_monitor_action_group.email.id + webhook_properties = null + }, + { + action_group_id = data.azurerm_monitor_action_group.slack.id + webhook_properties = null + }, + { + action_group_id = data.azurerm_monitor_action_group.opsgenie[0].id + webhook_properties = null + } + ] : [] + + private_dns_registration = var.pgres_flex_params.enable_private_dns_registration + private_dns_zone_name = "${var.env_short}.internal.postgresql.pagopa.it" + private_dns_zone_rg_name = data.azurerm_resource_group.rg_vnet.name + private_dns_record_cname = "gpd-db" +} + +resource "azurerm_postgresql_flexible_server_database" "apd_db_flex_test" { + count = var.env_short == "u" ? 1 : 0 + + name = var.gpd_db_name + server_id = module.postgres_flexible_server_private_test[0].id + collation = "en_US.utf8" + charset = "UTF8" +} + +resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_connection_test" { + count = var.env_short == "u" ? 1 : 0 + + name = "max_connections" + server_id = module.postgres_flexible_server_private_test[0].id + value = var.pgres_flex_params.max_connections +} + +# Message : FATAL: unsupported startup parameter: extra_float_digits +resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_ignore_startup_parameters_test" { + count = var.env_short == "u" ? 1 : 0 + + name = "pgbouncer.ignore_startup_parameters" + server_id = module.postgres_flexible_server_private_test[0].id + value = "extra_float_digits" +} + +resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_min_pool_size_test" { + count = var.env_short == "u" ? 1 : 0 + + name = "pgbouncer.min_pool_size" + server_id = module.postgres_flexible_server_private_test[0].id + value = var.env_short == "d" ? 1 : 10 +} + +# CDC https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-logical +resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_worker_process_test" { + count = var.env_short == "u" ? 1 : 0 + + name = "max_worker_processes" + server_id = module.postgres_flexible_server_private_test[0].id + value = var.pgres_flex_params.max_worker_process # var.env_short == "d" ? 16 : 32 +} + +resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_wal_level_test" { + count = var.pgres_flex_params.wal_level != null && var.env_short == "u" ? 1 : 0 + + name = "wal_level" + server_id = module.postgres_flexible_server_private_test[0].id + value = var.pgres_flex_params.wal_level # "logical", ... +} + +resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_shared_preoload_libraries_test" { + count = var.pgres_flex_params.wal_level != null && var.env_short == "u" ? 1 : 0 + + name = "shared_preload_libraries" + server_id = module.postgres_flexible_server_private_test[0].id + value = var.pgres_flex_params.shared_preoload_libraries # "pg_failover_slots" +} diff --git a/src/domains/gps-common/env/weu-prod/terraform.tfvars b/src/domains/gps-common/env/weu-prod/terraform.tfvars index f716ce39d3..ad682c40c0 100644 --- a/src/domains/gps-common/env/weu-prod/terraform.tfvars +++ b/src/domains/gps-common/env/weu-prod/terraform.tfvars @@ -64,7 +64,7 @@ gpd_upload_status_throughput = 10000 pgres_flex_params = { private_endpoint_enabled = true - sku_name = "GP_Standard_D8ds_v4" + sku_name = "GP_Standard_D16ds_v4" db_version = "13" # Possible values are 32768, 65536, 131072, 262144, 524288, 1048576, # 2097152, 4194304, 8388608, 16777216, and 33554432. diff --git a/src/domains/observability/05_app_forwarder.tf b/src/domains/observability/05_app_forwarder.tf index 531b100dda..3c811296bc 100644 --- a/src/domains/observability/05_app_forwarder.tf +++ b/src/domains/observability/05_app_forwarder.tf @@ -63,6 +63,8 @@ data "azurerm_resource_group" "rg_node_forwarder" { } data "azurerm_subnet" "subnet_node_forwarder" { + count = var.app_forwarder_enabled ? 1 : 0 + name = "pagopa-${var.env_short}-node-forwarder-snet" virtual_network_name = "pagopa-${var.env_short}-vnet" resource_group_name = "pagopa-${var.env_short}-vnet-rg" @@ -108,7 +110,7 @@ module "app_forwarder_app_service" { allowed_subnets = [data.azurerm_subnet.subnet_apim.id] allowed_ips = [] - subnet_id = data.azurerm_subnet.subnet_node_forwarder.id + subnet_id = data.azurerm_subnet.subnet_node_forwarder[0].id tags = var.tags } @@ -139,7 +141,7 @@ module "app_forwarder_slot_staging" { allowed_subnets = [data.azurerm_subnet.subnet_apim.id] allowed_ips = [] - subnet_id = data.azurerm_subnet.subnet_node_forwarder.id + subnet_id = data.azurerm_subnet.subnet_node_forwarder[0].id tags = var.tags } diff --git a/src/domains/observability/env/prod/terraform.tfvars b/src/domains/observability/env/prod/terraform.tfvars index 5bfe4f1a3d..2265a747ed 100644 --- a/src/domains/observability/env/prod/terraform.tfvars +++ b/src/domains/observability/env/prod/terraform.tfvars @@ -261,136 +261,67 @@ ehns_metric_alerts = { }, } - - - eventhubs_gpd = [ - { - name = "test-evh" # test - partitions = 1 - message_retention = 1 - consumers = ["test-evh"] + { + name = "gpd-ingestion.apd.payment_option" + partitions = 32 + message_retention = 7 + consumers = ["gpd-ingestion.apd.payment_option-rx-dl", ] keys = [ { - name = "test-evh" + name = "gpd-ingestion.apd.payment_option-rx-dl" listen = true + send = false + manage = false + }, + { + name = "gpd-ingestion.apd.payment_option-tx" + listen = false + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.payment_position" + partitions = 32 + message_retention = 7 + consumers = ["gpd-ingestion.apd.payment_position-rx-dl", ] + keys = [ + { + name = "gpd-ingestion.apd.payment_position-rx-dl" + listen = true + send = false + manage = false + }, + { + name = "gpd-ingestion.apd.payment_position-tx" + listen = false + send = true + manage = false + } + ] + }, + { + name = "gpd-ingestion.apd.transfer" + partitions = 32 + message_retention = 7 + consumers = ["gpd-ingestion.apd.transfer-rx-dl", ] + keys = [ + { + name = "gpd-ingestion.apd.transfer-rx-dl" + listen = true + send = false + manage = false + }, + { + name = "gpd-ingestion.apd.transfer-tx" + listen = false send = true manage = false } ] }, - # { - # name = "connect-cluster-offsets" # debezium internal use - # partitions = 32 - # message_retention = 7 - # consumers = ["connect-cluster-offsets"] - # keys = [ - # { - # name = "connect-cluster-offsets" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "connect-cluster-status" # debezium internal use - # partitions = 32 - # message_retention = 7 - # consumers = ["connect-cluster-offsets"] - # keys = [ - # { - # name = "connect-cluster-status" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "connect-cluster-configs" # debezium internal use - # partitions = 32 - # message_retention = 7 - # consumers = ["connect-cluster-configs"] - # keys = [ - # { - # name = "connect-cluster-configs" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.payment_option" - # partitions = 32 - # message_retention = 7 - # consumers = ["gpd-ingestion.apd.payment_option-rx-dl",] - # keys = [ - # { - # name = "gpd-ingestion.apd.payment_option-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.payment_option_metadata" - # partitions = 32 - # message_retention = 7 - # consumers = ["gpd-ingestion.apd.payment_option_metadata-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.payment_option_metadata-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.payment_position" - # partitions = 32 - # message_retention = 7 - # consumers = [ "gpd-ingestion.apd.payment_position-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.payment_position-rx-dl" - # listen = true - # send = true - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.transfer" - # partitions = 32 - # message_retention = 7 - # consumers = [ "gpd-ingestion.apd.transfer-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.transfer-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, - # { - # name = "gpd-ingestion.apd.transfer_metadata" - # partitions = 32 - # message_retention = 7 - # consumers = [ "gpd-ingestion.apd.transfer_metadata-rx-dl"] - # keys = [ - # { - # name = "gpd-ingestion.apd.transfer_metadata-rx-dl" - # listen = true - # send = false - # manage = false - # } - # ] - # }, ] From 333a70e186f979f2544bb1ae9e0556ddfcc30aae Mon Sep 17 00:00:00 2001 From: acialini Date: Fri, 15 Nov 2024 17:27:07 +0100 Subject: [PATCH 37/55] [PPANTT-186] feat: introduced alerts for ingestion manager functions --- src/domains/gps-app/00_alert_gpd_ingestion.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/domains/gps-app/00_alert_gpd_ingestion.tf b/src/domains/gps-app/00_alert_gpd_ingestion.tf index 8f609f3e05..3530da11dc 100644 --- a/src/domains/gps-app/00_alert_gpd_ingestion.tf +++ b/src/domains/gps-app/00_alert_gpd_ingestion.tf @@ -194,7 +194,7 @@ resource "azurerm_monitor_scheduled_query_rules_alert" "gpd-ingestion-manager-er for_each = { for c in local.fn_name_for_alerts_exceptions : c.name => c } resource_group_name = "dashboards" - name = "pagopa-${var.env_short}-gpd-ingestion-manager-error-alert" + name = "pagopa-${var.env_short}-gpd-ingestion-manager-error-alert-${each.value.id}" location = var.location action { From 8c1d635932cb704e8a4888aae1fe67b54f66c1fd Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 18 Nov 2024 14:07:15 +0100 Subject: [PATCH 38/55] fix --- src/domains/gps-common/how2_cdc_GPD.md | 4 ++++ .../gps-secret/secret/weu-prod/noedit_secret_enc.json | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/domains/gps-common/how2_cdc_GPD.md b/src/domains/gps-common/how2_cdc_GPD.md index d81372d1ec..f13f7be9ac 100644 --- a/src/domains/gps-common/how2_cdc_GPD.md +++ b/src/domains/gps-common/how2_cdc_GPD.md @@ -19,6 +19,10 @@ 1. apply https://github.com/pagopa/pagopa-infra/pull/2496 ( create/config GDP db + common secrets) NOTE : GDP network ( `+ Add 0.0.0.0 - 255.255.255.255` ) ( _only dev_ ) + ``` + src/domains/gps-secret + src/domains/gps-common + ``` 1. _[OPT iif not exists]_ user APD `./flyway_gpd.sh migrate -pagoPA apd apd -schemas=apd` diff --git a/src/domains/gps-secret/secret/weu-prod/noedit_secret_enc.json b/src/domains/gps-secret/secret/weu-prod/noedit_secret_enc.json index 6aecc70f1f..9221814b07 100644 --- a/src/domains/gps-secret/secret/weu-prod/noedit_secret_enc.json +++ b/src/domains/gps-secret/secret/weu-prod/noedit_secret_enc.json @@ -2,6 +2,7 @@ "config-cache-subscription-key": "ENC[AES256_GCM,data:Boy/LRbM7n6jeI3f8JIj77dSop9bmSYcClF5yOPnHP8=,iv:mYwuAQdw90B2tsXMiGLNrbDVu2RtXFwDU6Umc78DzKI=,tag:B1fh1tExQVpAd4ZpU0khWA==,type:str]", "cdc-logical-replication-apd-user": "ENC[AES256_GCM,data:2bYTx4PH,iv:NMEyW6KBgwEMmWdbBIkWcI+fn5fSJ3tmIr6BXtCQOCo=,tag:e9QnridAM2ih7Kw1x3e0Zg==,type:str]", "cdc-logical-replication-apd-pwd": "ENC[AES256_GCM,data:qocdlc1Z5Z4/F5F0fjGxJ0j4,iv:wEl471Vupsq00KkAeZ4iHddeXTiDCk/Noe4hKFW9wUw=,tag:S9yYq2mkq3bIBPTYsbTE5Q==,type:str]", + "tokenizer-api-key": "ENC[AES256_GCM,data:AVr5/g==,iv:DjYiP+y4W9WGiyBY5aNUiIRvK/4YQcFcoz5Bo7H7zwE=,tag:ffgLum7cTTXjrG4gR7zobg==,type:str]", "sops": { "kms": null, "gcp_kms": null, @@ -16,8 +17,8 @@ ], "hc_vault": null, "age": null, - "lastmodified": "2024-10-28T13:11:47Z", - "mac": "ENC[AES256_GCM,data:iJSkjohDJDQkqkjyIWSo4/Y2AJASYdkLJM7xKmG1RluZk3sRXE0SA1KVBagncjdHlqOw4RF1zFkuXZ2Ugab4/MfOx4QUuecOfI6nS+kX6K3UccCAPWTUe5BHPILVT8NLIohltf69/WTnDohCe6xFS0GHj5qGkFN5Mx96DODc3xo=,iv:9APBG5GmVIw5CQAbXFKvdgR0F0giQkvmoo1uOzE2TDY=,tag:auToUtKUardfNUNQHjRnxw==,type:str]", + "lastmodified": "2024-11-18T12:16:42Z", + "mac": "ENC[AES256_GCM,data:cOh5a+UBd/mKqUzXahvcFQxkOgnjx9+MJAQWaAhS9D++GUfszhYN/UsZctG8qsvWnqqDDRJKsqa+aTrxVKfTziau1fnDBWLFSA5aYynZDhoMr/e/RhzjDkh93P5QPz9mSPIciDWJKfIdUEaxWP/f7u41cLi4Q00+bF15KhYI9as=,iv:PuGnpWCQCuKlGDWZAc2MFGH6Mg8yS5mnj/GiUc+6kQI=,tag:DPY00xfqJxTNta3E2ot6uA==,type:str]", "pgp": null, "unencrypted_suffix": "_unencrypted", "version": "3.9.1" From d9907eb367329f180ad1680769b121829d7815bb Mon Sep 17 00:00:00 2001 From: acialini Date: Mon, 18 Nov 2024 16:31:52 +0100 Subject: [PATCH 39/55] [PPANTT-185] feat: introduced logging --- src/domains/gps-app/00_alert_gpd_ingestion.tf | 2 ++ src/domains/gps-app/05_debezium_connect.tf | 25 +++++++++++++------ src/domains/gps-app/yaml/kafka-connect.yaml | 5 ++++ src/elk-monitoring/05_elastic_stack.tf | 2 +- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/domains/gps-app/00_alert_gpd_ingestion.tf b/src/domains/gps-app/00_alert_gpd_ingestion.tf index 3530da11dc..a2f39acafb 100644 --- a/src/domains/gps-app/00_alert_gpd_ingestion.tf +++ b/src/domains/gps-app/00_alert_gpd_ingestion.tf @@ -223,3 +223,5 @@ resource "azurerm_monitor_scheduled_query_rules_alert" "gpd-ingestion-manager-er } } + + diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index b7201357cd..9b33549e48 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -11,6 +11,12 @@ data "azurerm_key_vault_secret" "pgres_gpd_cdc_pwd" { key_vault_id = data.azurerm_key_vault.kv.id } +data "azurerm_key_vault_secret" "otel_headers" { + # name = "db-apd-user-password" + name = "otel_headers" + key_vault_id = data.azurerm_key_vault.kv.id +} + data "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" { name = "cdc-gpd-connection-string" namespace_name = "pagopa-${var.env_short}-itn-observ-gpd-evh" @@ -59,14 +65,17 @@ locals { # https://learn.microsoft.com/it-it/azure/event-hubs/event-hubs-kafka-connect-debezium#configure-kafka-connect-for-event-hubs kafka_connect_yaml = templatefile("${path.module}/yaml/kafka-connect.yaml", { - namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - replicas = var.replicas - request_memory = var.request_memory - request_cpu = var.request_cpu - limits_memory = var.limits_memory - limits_cpu = var.limits_cpu - bootstrap_servers = "pagopa-${var.env_short}-itn-observ-gpd-evh.servicebus.windows.net:9093" - container_registry = var.container_registry + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + replicas = var.replicas + request_memory = var.request_memory + request_cpu = var.request_cpu + limits_memory = var.limits_memory + limits_cpu = var.limits_cpu + bootstrap_servers = "pagopa-${var.env_short}-itn-observ-gpd-evh.servicebus.windows.net:9093" + container_registry = var.container_registry + otlp_endpoint = "http://otel-collector.elastic-system.svc:4317" + otlp_resource_attributes = "service.name=gpddebeziumconnectorkotl,deployment.environment=${var.env}" + otlp_headers = data.azurerm_key_vault_secret.otel_headers.value }) postgres_connector_yaml = templatefile("${path.module}/yaml/postgres-connector.yaml", { diff --git a/src/domains/gps-app/yaml/kafka-connect.yaml b/src/domains/gps-app/yaml/kafka-connect.yaml index c266bd9527..d08dfacd3e 100644 --- a/src/domains/gps-app/yaml/kafka-connect.yaml +++ b/src/domains/gps-app/yaml/kafka-connect.yaml @@ -45,6 +45,11 @@ spec: internal.value.converter: org.apache.kafka.connect.json.JsonConverter internal.key.converter.schemas.enable: false internal.value.converter.schemas.enable: false + otel.traces.exporter: otlp + otel.propagators: tracecontext + otel.exporter.otlp.endpoint: "${otlp_endpoint}" + otel.resource.attributes: "${otlp_resource_attributes}" + otel.expoerter.oltp.header: "${otlp_key_headers}" build: output: type: docker diff --git a/src/elk-monitoring/05_elastic_stack.tf b/src/elk-monitoring/05_elastic_stack.tf index 2a90dd3490..1ff5fe3400 100644 --- a/src/elk-monitoring/05_elastic_stack.tf +++ b/src/elk-monitoring/05_elastic_stack.tf @@ -97,7 +97,7 @@ module "elastic_stack" { /* apiconfig */ "pagopaapiconfig-postgresql", "pagopaapiconfig-oracle", "apiconfig-selfcare-integration-microservice-chart", "cache-oracle", "cache-postgresql", "cache-replica-oracle", "cache-replica-postgresql", /* ecommerce */ "pagopaecommerceeventdispatcherservice-microservice-chart", "pagopaecommercepaymentmethodsservice-microservice-chart", "pagopaecommercepaymentrequestsservice-microservice-chart", "pagopaecommercetransactionsservice-microservice-chart", "pagopaecommercetxschedulerservice-microservice-chart", "pagopanotificationsservice-microservice-chart", /* selfcare */ "pagopaselfcaremsbackofficebackend-microservice-chart", "backoffice-external", - /* gps */ "gpd-core-microservice-chart", "pagopagpdpayments-microservice-chart", "pagopareportingorgsenrollment-microservice-chart", "pagopaspontaneouspayments-microservice-chart", "gpd-payments-pull", "gpd-upload-microservice-chart" + /* gps */ "gpd-core-microservice-chart", "pagopagpdpayments-microservice-chart", "pagopareportingorgsenrollment-microservice-chart", "pagopaspontaneouspayments-microservice-chart", "gpd-payments-pull", "gpd-upload-microservice-chart", "pagopapagopagpdingestionmanager-microservice-chart" ] eck_license = file("${path.module}/env/eck_license/pagopa-spa-4a1285e5-9c2c-4f9f-948a-9600095edc2f-orchestration.json") From 1a643746d97f5b56a370226b6cb9ef900f6ebedc Mon Sep 17 00:00:00 2001 From: acialini Date: Mon, 18 Nov 2024 17:59:53 +0100 Subject: [PATCH 40/55] [PPANTT-168] feat: Updated postgres flex config --- ...ST_2_DELETE.tf => 03_postgresql_gpd_db.tf} | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) rename src/domains/gps-common/{03_postgresql_gpd_TEST_2_DELETE.tf => 03_postgresql_gpd_db.tf} (83%) diff --git a/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf b/src/domains/gps-common/03_postgresql_gpd_db.tf similarity index 83% rename from src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf rename to src/domains/gps-common/03_postgresql_gpd_db.tf index 090f78f851..452c77f11d 100644 --- a/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf +++ b/src/domains/gps-common/03_postgresql_gpd_db.tf @@ -1,4 +1,4 @@ -# REMOVE IT after close MS issue +# REMOVE IT after close MS issue # Support Request: certificates-do-not-conform-to-algorithm # ########################################################## # ########################################################## @@ -6,11 +6,11 @@ # ########################################################## # https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-compare-single-server-flexible-server -module "postgres_flexible_server_private_test" { # private only into UAT and PROD env +module "postgres_flexible_server_private" { # private only into UAT and PROD env source = "./.terraform/modules/__v3__/postgres_flexible_server" - count = var.env_short == "u" ? 1 : 0 + count = var.env_short == "u" ? 1 : 0 - name = format("%s-gpd-pgflex-test", local.product) + name = format("%s-%s-gpd-pgflex",local.product, var.location_short) location = azurerm_resource_group.flex_data[0].location resource_group_name = azurerm_resource_group.flex_data[0].name @@ -66,46 +66,46 @@ module "postgres_flexible_server_private_test" { # private only into UAT and PRO private_dns_record_cname = "gpd-db" } -resource "azurerm_postgresql_flexible_server_database" "apd_db_flex_test" { - count = var.env_short == "u" ? 1 : 0 +resource "azurerm_postgresql_flexible_server_database" "apd_db_flex" { + count = var.env_short == "u" ? 1 : 0 name = var.gpd_db_name - server_id = module.postgres_flexible_server_private_test[0].id + server_id = module.postgres_flexible_server_private[0].id collation = "en_US.utf8" charset = "UTF8" } -resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_connection_test" { - count = var.env_short == "u" ? 1 : 0 +resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_connection" { + count = var.env_short == "u" ? 1 : 0 name = "max_connections" - server_id = module.postgres_flexible_server_private_test[0].id + server_id = module.postgres_flexible_server_private[0].id value = var.pgres_flex_params.max_connections } # Message : FATAL: unsupported startup parameter: extra_float_digits resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_ignore_startup_parameters_test" { - count = var.env_short == "u" ? 1 : 0 + count = var.env_short == "u" ? 1 : 0 name = "pgbouncer.ignore_startup_parameters" - server_id = module.postgres_flexible_server_private_test[0].id + server_id = module.postgres_flexible_server_private[0].id value = "extra_float_digits" } resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_min_pool_size_test" { - count = var.env_short == "u" ? 1 : 0 + count = var.env_short == "u" ? 1 : 0 name = "pgbouncer.min_pool_size" - server_id = module.postgres_flexible_server_private_test[0].id + server_id = module.postgres_flexible_server_private[0].id value = var.env_short == "d" ? 1 : 10 } # CDC https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-logical resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_worker_process_test" { - count = var.env_short == "u" ? 1 : 0 + count = var.env_short == "u" ? 1 : 0 name = "max_worker_processes" - server_id = module.postgres_flexible_server_private_test[0].id + server_id = module.postgres_flexible_server_private[0].id value = var.pgres_flex_params.max_worker_process # var.env_short == "d" ? 16 : 32 } @@ -113,7 +113,7 @@ resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_wal_lev count = var.pgres_flex_params.wal_level != null && var.env_short == "u" ? 1 : 0 name = "wal_level" - server_id = module.postgres_flexible_server_private_test[0].id + server_id = module.postgres_flexible_server_private[0].id value = var.pgres_flex_params.wal_level # "logical", ... } @@ -121,6 +121,6 @@ resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_shared_ count = var.pgres_flex_params.wal_level != null && var.env_short == "u" ? 1 : 0 name = "shared_preload_libraries" - server_id = module.postgres_flexible_server_private_test[0].id + server_id = module.postgres_flexible_server_private[0].id value = var.pgres_flex_params.shared_preoload_libraries # "pg_failover_slots" } From 7cb32592d11e0a0a0ce893e82566f6258cd96e34 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Tue, 19 Nov 2024 11:46:57 +0100 Subject: [PATCH 41/55] fix new name gpd db --- .../gps-common/03_postgresql_gpd_db.tf | 45 +++++++------------ src/domains/gps-common/how2_cdc_GPD.md | 8 +--- 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/src/domains/gps-common/03_postgresql_gpd_db.tf b/src/domains/gps-common/03_postgresql_gpd_db.tf index 452c77f11d..8fe147474e 100644 --- a/src/domains/gps-common/03_postgresql_gpd_db.tf +++ b/src/domains/gps-common/03_postgresql_gpd_db.tf @@ -6,9 +6,8 @@ # ########################################################## # https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-compare-single-server-flexible-server -module "postgres_flexible_server_private" { # private only into UAT and PROD env +module "postgres_flexible_server_private_db" { source = "./.terraform/modules/__v3__/postgres_flexible_server" - count = var.env_short == "u" ? 1 : 0 name = format("%s-%s-gpd-pgflex",local.product, var.location_short) @@ -66,61 +65,51 @@ module "postgres_flexible_server_private" { # private only into UAT and PROD env private_dns_record_cname = "gpd-db" } -resource "azurerm_postgresql_flexible_server_database" "apd_db_flex" { - count = var.env_short == "u" ? 1 : 0 - +resource "azurerm_postgresql_flexible_server_database" "pg_charset" { name = var.gpd_db_name - server_id = module.postgres_flexible_server_private[0].id + server_id = module.postgres_flexible_server_private_db.id collation = "en_US.utf8" charset = "UTF8" } -resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_connection" { - count = var.env_short == "u" ? 1 : 0 - +resource "azurerm_postgresql_flexible_server_configuration" "pg_max_connections" { name = "max_connections" - server_id = module.postgres_flexible_server_private[0].id + server_id = module.postgres_flexible_server_private_db.id value = var.pgres_flex_params.max_connections } # Message : FATAL: unsupported startup parameter: extra_float_digits -resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_ignore_startup_parameters_test" { - count = var.env_short == "u" ? 1 : 0 - +resource "azurerm_postgresql_flexible_server_configuration" "pd_pgbouncer_ignore_startup_parameters" { name = "pgbouncer.ignore_startup_parameters" - server_id = module.postgres_flexible_server_private[0].id + server_id = module.postgres_flexible_server_private_db.id value = "extra_float_digits" } -resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_min_pool_size_test" { - count = var.env_short == "u" ? 1 : 0 - +resource "azurerm_postgresql_flexible_server_configuration" "pg_pgbouncer_min_pool_size" { name = "pgbouncer.min_pool_size" - server_id = module.postgres_flexible_server_private[0].id + server_id = module.postgres_flexible_server_private_db.id value = var.env_short == "d" ? 1 : 10 } # CDC https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-logical -resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_worker_process_test" { - count = var.env_short == "u" ? 1 : 0 - +resource "azurerm_postgresql_flexible_server_configuration" "pg_max_worker_processes" { name = "max_worker_processes" - server_id = module.postgres_flexible_server_private[0].id + server_id = module.postgres_flexible_server_private_db.id value = var.pgres_flex_params.max_worker_process # var.env_short == "d" ? 16 : 32 } -resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_wal_level_test" { - count = var.pgres_flex_params.wal_level != null && var.env_short == "u" ? 1 : 0 +resource "azurerm_postgresql_flexible_server_configuration" "pg_wal_level" { + count = var.pgres_flex_params.wal_level != null ? 1 : 0 name = "wal_level" - server_id = module.postgres_flexible_server_private[0].id + server_id = module.postgres_flexible_server_private_db.id value = var.pgres_flex_params.wal_level # "logical", ... } -resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_shared_preoload_libraries_test" { - count = var.pgres_flex_params.wal_level != null && var.env_short == "u" ? 1 : 0 +resource "azurerm_postgresql_flexible_server_configuration" "pg_shared_preload_libraries" { + count = var.pgres_flex_params.wal_level != null ? 1 : 0 name = "shared_preload_libraries" - server_id = module.postgres_flexible_server_private[0].id + server_id = module.postgres_flexible_server_private_db.id value = var.pgres_flex_params.shared_preoload_libraries # "pg_failover_slots" } diff --git a/src/domains/gps-common/how2_cdc_GPD.md b/src/domains/gps-common/how2_cdc_GPD.md index f13f7be9ac..a0f1f3a298 100644 --- a/src/domains/gps-common/how2_cdc_GPD.md +++ b/src/domains/gps-common/how2_cdc_GPD.md @@ -17,12 +17,8 @@ + `src/domains/observability/gpd_evh_create__az.sh` for eventhub with `cleanup-policy` -1. apply https://github.com/pagopa/pagopa-infra/pull/2496 ( create/config GDP db + common secrets) - NOTE : GDP network ( `+ Add 0.0.0.0 - 255.255.255.255` ) ( _only dev_ ) - ``` - src/domains/gps-secret - src/domains/gps-common - ``` +1. apply secrets `src/domains/gps-secret` + apply DB `src/domains/gps-common` ⚠️⚠️ _ReCreate DB GPD with new name convection_⚠️⚠️ `pagopa---gpd-pgflex 1. _[OPT iif not exists]_ user APD `./flyway_gpd.sh migrate -pagoPA apd apd -schemas=apd` From f4c7310ca9bdef4e2b03afe97bb3cbdce539f4fe Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Tue, 19 Nov 2024 11:50:54 +0100 Subject: [PATCH 42/55] fix new name gpd db --- src/domains/gps-common/how2_cdc_GPD.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/domains/gps-common/how2_cdc_GPD.md b/src/domains/gps-common/how2_cdc_GPD.md index a0f1f3a298..bf4d0bc333 100644 --- a/src/domains/gps-common/how2_cdc_GPD.md +++ b/src/domains/gps-common/how2_cdc_GPD.md @@ -18,7 +18,11 @@ + `src/domains/observability/gpd_evh_create__az.sh` for eventhub with `cleanup-policy` 1. apply secrets `src/domains/gps-secret` - apply DB `src/domains/gps-common` ⚠️⚠️ _ReCreate DB GPD with new name convection_⚠️⚠️ `pagopa---gpd-pgflex + apply DB `src/domains/gps-common` + + ⚠️⚠️ _ReCreate DB GPD with new name convection_ ⚠️⚠️ + + `pagopa---gpd-pgflex` 1. _[OPT iif not exists]_ user APD `./flyway_gpd.sh migrate -pagoPA apd apd -schemas=apd` From a45f996b170b2c40fa02d3d8eca8fb60aed2c846 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Tue, 19 Nov 2024 12:58:51 +0100 Subject: [PATCH 43/55] fix --- src/domains/gps-common/99_locals.tf | 2 +- src/domains/gps-common/how2_cdc_GPD.md | 2 +- src/psql/flyway_gpd.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/domains/gps-common/99_locals.tf b/src/domains/gps-common/99_locals.tf index afabdfc92c..f20061da06 100644 --- a/src/domains/gps-common/99_locals.tf +++ b/src/domains/gps-common/99_locals.tf @@ -41,7 +41,7 @@ locals { # gpd_hostname = var.env_short != "d" ? module.postgres_flexible_server_private.fqdn : module.postgresql[0].fqdn # gpd_dbmsport = var.env_short != "d" ? "6432" : "5432" - gpd_hostname = module.postgres_flexible_server_private[0].fqdn + gpd_hostname = var.env_short == "p" ? module.postgres_flexible_server_private[0].fqdn : module.postgres_flexible_server_private_db.fqdn gpd_dbmsport = "6432" azdo_managed_identity_rg_name = "pagopa-${var.env_short}-identity-rg" diff --git a/src/domains/gps-common/how2_cdc_GPD.md b/src/domains/gps-common/how2_cdc_GPD.md index bf4d0bc333..9ba09b9efe 100644 --- a/src/domains/gps-common/how2_cdc_GPD.md +++ b/src/domains/gps-common/how2_cdc_GPD.md @@ -20,7 +20,7 @@ 1. apply secrets `src/domains/gps-secret` apply DB `src/domains/gps-common` - ⚠️⚠️ _ReCreate DB GPD with new name convection_ ⚠️⚠️ + ⚠️⚠️ _ReCreate DB GPD with new name convention_ ⚠️⚠️ `pagopa---gpd-pgflex` diff --git a/src/psql/flyway_gpd.sh b/src/psql/flyway_gpd.sh index 118db25097..e6a6c4ce4b 100755 --- a/src/psql/flyway_gpd.sh +++ b/src/psql/flyway_gpd.sh @@ -51,8 +51,8 @@ printf "Resource Group Name: %s\n" "${resource_group_name}" printf ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> 1\n" # flexible-server -psql_server_name=$(az postgres flexible-server list -o tsv --query "[?contains(name,'pgflex')].{Name:name}" | grep gpd | head -1) -psql_server_private_fqdn=$(az postgres flexible-server list -o tsv --query "[?contains(name,'pgflex')].{Name:fullyQualifiedDomainName}" | grep gpd | head -1) +psql_server_name=$(az postgres flexible-server list -o tsv --query "[?contains(name,'pgflex')].{Name:name}" | grep "weu-gpd" | head -1) +psql_server_private_fqdn=$(az postgres flexible-server list -o tsv --query "[?contains(name,'pgflex')].{Name:fullyQualifiedDomainName}" | grep "weu-gpd" | head -1) # kv keyvault_name=$(az keyvault list -o tsv --query "[?contains(name,'gps')].{Name:name}") From f10ca806e2c78ff33770465ebf76ba96c085ed0c Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Tue, 19 Nov 2024 14:30:28 +0100 Subject: [PATCH 44/55] fix --- src/domains/observability/02_security.tf | 4 ++++ .../observability/03_eventhub_msg_gdp.tf | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/src/domains/observability/02_security.tf b/src/domains/observability/02_security.tf index aaf6dc53b7..dbbd130851 100644 --- a/src/domains/observability/02_security.tf +++ b/src/domains/observability/02_security.tf @@ -2,3 +2,7 @@ data "azurerm_key_vault" "kv" { name = "${local.product}-shared-kv" resource_group_name = "${local.product}-shared-sec-rg" } +data "azurerm_key_vault" "gps_kv" { + name = "${local.product}-gps-kv" + resource_group_name = "${local.product}-gps-sec-rg" +} \ No newline at end of file diff --git a/src/domains/observability/03_eventhub_msg_gdp.tf b/src/domains/observability/03_eventhub_msg_gdp.tf index 893b86d984..98b73fc8cf 100644 --- a/src/domains/observability/03_eventhub_msg_gdp.tf +++ b/src/domains/observability/03_eventhub_msg_gdp.tf @@ -77,3 +77,25 @@ resource "azurerm_eventhub_namespace_authorization_rule" "cdc_connection_string" # --partition-count 1 \ # --retention-time 24 + +resource "azurerm_eventhub_namespace_authorization_rule" "cdc_test_connection_string" { + count = var.env != "p" ? 1 : 0 + + name = "cdc-gpd-test-connection-string" + namespace_name = module.eventhub_namespace_observability_gpd.name + resource_group_name = azurerm_resource_group.eventhub_observability_rg.name + listen = true + send = true + manage = false +} + +resource "azurerm_key_vault_secret" "azure_web_jobs_storage_kv" { + count = var.env != "p" ? 1 : 0 + + name = "cdc-gpd-test-connection-string" + value = azurerm_eventhub_namespace_authorization_rule.cdc_test_connection_string[0].primary_connection_string + content_type = "text/plain" + key_vault_id = data.azurerm_key_vault.gps_kv.id +} + + From a4694f7b9a4bc3572203e8ed4ed26737adf56a2d Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Thu, 21 Nov 2024 11:20:58 +0100 Subject: [PATCH 45/55] fix --- src/domains/gps-app/05_debezium_connect.tf | 5 +- src/domains/gps-app/yaml/kafka-connect.yaml | 2 +- src/domains/gps-common/how2_cdc_GPD.md | 7 ++ .../observability/gpd_evh_create__az.sh | 68 +++++++++---------- 4 files changed, 44 insertions(+), 38 deletions(-) diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index 9b33549e48..3d344812ef 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -12,8 +12,7 @@ data "azurerm_key_vault_secret" "pgres_gpd_cdc_pwd" { } data "azurerm_key_vault_secret" "otel_headers" { - # name = "db-apd-user-password" - name = "otel_headers" + name = "elastic-apm-secret-token" key_vault_id = data.azurerm_key_vault.kv.id } @@ -80,7 +79,7 @@ locals { postgres_connector_yaml = templatefile("${path.module}/yaml/postgres-connector.yaml", { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name - postgres_hostname = "pagopa-${var.env_short}-gpd-pgflex.postgres.database.azure.com" + postgres_hostname = "pagopa-${var.env_short}-${var.location_short}-gpd-pgflex.postgres.database.azure.com" postgres_port = 5432 postgres_db_name = var.postgres_db_name diff --git a/src/domains/gps-app/yaml/kafka-connect.yaml b/src/domains/gps-app/yaml/kafka-connect.yaml index d08dfacd3e..9c4a02a434 100644 --- a/src/domains/gps-app/yaml/kafka-connect.yaml +++ b/src/domains/gps-app/yaml/kafka-connect.yaml @@ -49,7 +49,7 @@ spec: otel.propagators: tracecontext otel.exporter.otlp.endpoint: "${otlp_endpoint}" otel.resource.attributes: "${otlp_resource_attributes}" - otel.expoerter.oltp.header: "${otlp_key_headers}" + otel.expoerter.oltp.header: "${otlp_headers}" build: output: type: docker diff --git a/src/domains/gps-common/how2_cdc_GPD.md b/src/domains/gps-common/how2_cdc_GPD.md index 9ba09b9efe..27766a6749 100644 --- a/src/domains/gps-common/how2_cdc_GPD.md +++ b/src/domains/gps-common/how2_cdc_GPD.md @@ -65,6 +65,13 @@ kubectl config use-context ``` + ``` + sh set_registry_secrets.sh \ + pagopacommonacr.azurecr.io \ + pagopacommonacr \ + + ``` + 1. deploy debezium `src/domains/gps-app` ```sh diff --git a/src/domains/observability/gpd_evh_create__az.sh b/src/domains/observability/gpd_evh_create__az.sh index 992f38b6cc..9254066847 100644 --- a/src/domains/observability/gpd_evh_create__az.sh +++ b/src/domains/observability/gpd_evh_create__az.sh @@ -6,46 +6,46 @@ set -e # config topics # ============================================================== +env=$1 + + +echo ">>>>>> 1" + +az eventhubs eventhub create \ +-g pagopa-$env-itn-observ-evh-rg \ +-n "connect-cluster-offsets" \ +--namespace-name pagopa-$env-itn-observ-gpd-evh \ +--cleanup-policy "Compact" \ +--status "Active" \ +--partition-count 1 \ +--retention-time 24 + +echo ">>>>>> 2" + +az eventhubs eventhub create \ +-g pagopa-$env-itn-observ-evh-rg \ +-n "connect-cluster-status" \ +--namespace-name pagopa-$env-itn-observ-gpd-evh \ +--cleanup-policy "Compact" \ +--status "Active" \ +--partition-count 1 \ +--retention-time 24 -# echo ">>>>>> 1" - -# az eventhubs eventhub create \ -# -g pagopa-$env-itn-observ-evh-rg \ -# -n "connect-cluster-offsets" \ -# --namespace-name pagopa-$env-itn-observ-gpd-evh \ -# --cleanup-policy "Compact" \ -# --status "Active" \ -# --partition-count 1 \ -# --retention-time 24 - -# echo ">>>>>> 2" - -# az eventhubs eventhub create \ -# -g pagopa-$env-itn-observ-evh-rg \ -# -n "connect-cluster-status" \ -# --namespace-name pagopa-$env-itn-observ-gpd-evh \ -# --cleanup-policy "Compact" \ -# --status "Active" \ -# --partition-count 1 \ -# --retention-time 24 - -# echo ">>>>>> 3" - -# az eventhubs eventhub create \ -# -g pagopa-$env-itn-observ-evh-rg \ -# -n "connect-cluster-configs" \ -# --namespace-name pagopa-$env-itn-observ-gpd-evh \ -# --cleanup-policy "Compact" \ -# --status "Active" \ -# --partition-count 1 \ -# --retention-time 24 +echo ">>>>>> 3" + +az eventhubs eventhub create \ +-g pagopa-$env-itn-observ-evh-rg \ +-n "connect-cluster-configs" \ +--namespace-name pagopa-$env-itn-observ-gpd-evh \ +--cleanup-policy "Compact" \ +--status "Active" \ +--partition-count 1 \ +--retention-time 24 # ============================================================== # logical topics # ============================================================== -env=$1 - echo ">>>>>> 4" az eventhubs eventhub create \ From b0dbf87ee1db5e4dbf15e5d9591bf58287d5f8e0 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Mon, 25 Nov 2024 19:24:42 +0100 Subject: [PATCH 46/55] fix --- .../03_postgresql_gpd_TEST_2_DELETE.tf | 124 ------------------ .../03_postgresql_gpd_single_server.tf | 73 ----------- src/domains/gps-common/99_locals.tf | 2 - 3 files changed, 199 deletions(-) delete mode 100644 src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf delete mode 100644 src/domains/gps-common/03_postgresql_gpd_single_server.tf diff --git a/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf b/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf deleted file mode 100644 index b6f6bded62..0000000000 --- a/src/domains/gps-common/03_postgresql_gpd_TEST_2_DELETE.tf +++ /dev/null @@ -1,124 +0,0 @@ -## REMOVE IT after close MS issue -## Support Request: certificates-do-not-conform-to-algorithm -## ########################################################## -## ########################################################## -## ########################################################## -## ########################################################## - -# # https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-compare-single-server-flexible-server -# module "postgres_flexible_server_private_test" { # private only into UAT and PROD env -# source = "./.terraform/modules/__v3__/postgres_flexible_server" -# count = var.env_short == "u" ? 1 : 0 - -# name = format("%s-gpd-pgflex-test", local.product) - -# location = azurerm_resource_group.flex_data[0].location -# resource_group_name = azurerm_resource_group.flex_data[0].name - -# ### Network -# private_endpoint_enabled = var.pgres_flex_params.private_endpoint_enabled -# private_dns_zone_id = var.env_short != "d" ? data.azurerm_private_dns_zone.postgres[0].id : null -# delegated_subnet_id = module.postgres_flexible_snet[0].id -# public_network_access_enabled = var.pgres_flex_params.public_network_access_enabled - -# ### admin credentials -# administrator_login = data.azurerm_key_vault_secret.pgres_admin_login.value -# administrator_password = data.azurerm_key_vault_secret.pgres_admin_pwd.value - -# sku_name = var.pgres_flex_params.sku_name -# db_version = var.pgres_flex_params.db_version -# storage_mb = var.pgres_flex_params.storage_mb -# zone = var.pgres_flex_params.zone -# backup_retention_days = var.pgres_flex_params.backup_retention_days -# create_mode = null // the update of this argument triggers a replace -# geo_redundant_backup_enabled = var.pgres_flex_params.geo_redundant_backup_enabled - -# high_availability_enabled = var.pgres_flex_params.high_availability_enabled -# standby_availability_zone = var.pgres_flex_params.standby_availability_zone -# pgbouncer_enabled = var.pgres_flex_params.pgbouncer_enabled - -# diagnostic_settings_enabled = false - -# tags = var.tags - -# # alert section -# custom_metric_alerts = var.pgres_flex_params.alerts_enabled ? var.pgflex_public_metric_alerts : {} -# alerts_enabled = var.pgres_flex_params.alerts_enabled - -# alert_action = var.pgres_flex_params.alerts_enabled ? [ -# { -# action_group_id = data.azurerm_monitor_action_group.email.id -# webhook_properties = null -# }, -# { -# action_group_id = data.azurerm_monitor_action_group.slack.id -# webhook_properties = null -# }, -# { -# action_group_id = data.azurerm_monitor_action_group.opsgenie[0].id -# webhook_properties = null -# } -# ] : [] - -# private_dns_registration = var.pgres_flex_params.enable_private_dns_registration -# private_dns_zone_name = "${var.env_short}.internal.postgresql.pagopa.it" -# private_dns_zone_rg_name = data.azurerm_resource_group.rg_vnet.name -# private_dns_record_cname = "gpd-db" -# } - -# resource "azurerm_postgresql_flexible_server_database" "apd_db_flex_test" { -# count = 1 # forced - -# name = var.gpd_db_name -# server_id = module.postgres_flexible_server_private_test[0].id -# collation = "en_US.utf8" -# charset = "UTF8" -# } - -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_connection_test" { -# count = 1 # forced - -# name = "max_connections" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = var.pgres_flex_params.max_connections -# } - -# # Message : FATAL: unsupported startup parameter: extra_float_digits -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_ignore_startup_parameters_test" { -# count = 1 # forced - -# name = "pgbouncer.ignore_startup_parameters" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = "extra_float_digits" -# } - -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_min_pool_size_test" { -# count = 1 # forced - -# name = "pgbouncer.min_pool_size" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = var.env_short == "d" ? 1 : 10 -# } - -# # CDC https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-logical -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_worker_process_test" { -# name = "max_worker_processes" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = var.pgres_flex_params.max_worker_process # var.env_short == "d" ? 16 : 32 -# } - -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_wal_level_test" { -# count = var.pgres_flex_params.wal_level != null ? 1 : 0 - -# name = "wal_level" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = var.pgres_flex_params.wal_level # "logical", ... -# } - -# resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_shared_preoload_libraries_test" { -# count = var.pgres_flex_params.wal_level != null ? 1 : 0 - -# name = "shared_preload_libraries" -# server_id = module.postgres_flexible_server_private_test[0].id -# value = var.pgres_flex_params.shared_preoload_libraries # "pg_failover_slots" -# } diff --git a/src/domains/gps-common/03_postgresql_gpd_single_server.tf b/src/domains/gps-common/03_postgresql_gpd_single_server.tf deleted file mode 100644 index 97453796b1..0000000000 --- a/src/domains/gps-common/03_postgresql_gpd_single_server.tf +++ /dev/null @@ -1,73 +0,0 @@ -# # ######################################################################################################################## -# # ########################################### POSTGRES DEV ############################################################### -# # ######################################################################################################################## -# # # ⚠️⚠️ Azure Database for PostgreSQL Single Server retires in March 2025. -# # # ⚠️⚠️ This server will automatically migrate to Azure Database for PostgreSQL Flexible Server in 2 days. Learn More - -# module "postgresql_snet" { -# count = var.env_short == "d" ? 1 : 0 -# source = "./.terraform/modules/__v3__/subnet" - -# name = format("%s-gpd-postgresql-snet", local.product) -# address_prefixes = var.cidr_subnet_pg_singleser -# resource_group_name = local.vnet_resource_group_name -# virtual_network_name = local.vnet_name -# service_endpoints = ["Microsoft.Sql"] -# private_endpoint_network_policies_enabled = false - -# delegation = { -# name = "delegation" -# service_delegation = { -# name = "Microsoft.ContainerInstance/containerGroups" -# actions = ["Microsoft.Network/virtualNetworks/subnets/action"] -# } -# } -# } - -# #tfsec:ignore:azure-database-no-public-access -# module "postgresql" { -# count = var.env_short == "d" ? 1 : 0 -# source = "./.terraform/modules/__v3__/postgresql_server" - -# name = format("%s-gpd-postgresql", local.product) -# location = azurerm_resource_group.gpd_rg.location -# resource_group_name = azurerm_resource_group.gpd_rg.name - -# administrator_login = data.azurerm_key_vault_secret.pgres_admin_login.value -# administrator_login_password = data.azurerm_key_vault_secret.pgres_admin_pwd.value - -# sku_name = "B_Gen5_1" -# db_version = 11 -# geo_redundant_backup_enabled = false - -# public_network_access_enabled = false -# network_rules = var.postgresql_network_rules - -# private_endpoint = { -# enabled = false -# virtual_network_id = data.azurerm_virtual_network.vnet.id -# subnet_id = module.postgresql_snet[0].id -# private_dns_zone_ids = [] -# } - -# enable_replica = false -# alerts_enabled = false -# lock_enable = false - -# tags = var.tags -# } - -# resource "azurerm_postgresql_database" "apd_db" { -# count = var.env_short == "d" ? 1 : 0 -# name = var.gpd_db_name -# resource_group_name = azurerm_resource_group.gpd_rg.name -# server_name = module.postgresql[0].name -# charset = "UTF8" -# collation = "it_IT" # "Italian_Italy.1252" - -# lifecycle { -# ignore_changes = [ -# collation, -# ] -# } -# } diff --git a/src/domains/gps-common/99_locals.tf b/src/domains/gps-common/99_locals.tf index f20061da06..fcac481c02 100644 --- a/src/domains/gps-common/99_locals.tf +++ b/src/domains/gps-common/99_locals.tf @@ -39,8 +39,6 @@ locals { aks_subnet_name = "${var.prefix}-${var.env_short}-${var.location_short}-${var.env}-aks-snet" azdo_subnet_name = "${local.product}-azdoa-snet" - # gpd_hostname = var.env_short != "d" ? module.postgres_flexible_server_private.fqdn : module.postgresql[0].fqdn - # gpd_dbmsport = var.env_short != "d" ? "6432" : "5432" gpd_hostname = var.env_short == "p" ? module.postgres_flexible_server_private[0].fqdn : module.postgres_flexible_server_private_db.fqdn gpd_dbmsport = "6432" From e89dab98c25ac647d4dc661d15ecfe98913297f5 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Wed, 27 Nov 2024 12:58:40 +0100 Subject: [PATCH 47/55] fix --- src/domains/gps-common/03_postgresql_gpd.tf | 16 +++++++++------- .../gps-common/env/weu-dev/terraform.tfvars | 2 +- .../gps-common/env/weu-uat/terraform.tfvars | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/domains/gps-common/03_postgresql_gpd.tf b/src/domains/gps-common/03_postgresql_gpd.tf index ab280b4cb6..067775d43d 100644 --- a/src/domains/gps-common/03_postgresql_gpd.tf +++ b/src/domains/gps-common/03_postgresql_gpd.tf @@ -59,7 +59,7 @@ data "azurerm_private_dns_zone" "postgres" { # https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-compare-single-server-flexible-server module "postgres_flexible_server_private" { # private only into UAT and PROD env source = "./.terraform/modules/__v3__/postgres_flexible_server" - count = 1 # forced + count = var.env_short != "d" ? 1 : 0 # forced name = format("%s-gpd-pgflex", local.product) @@ -118,7 +118,7 @@ module "postgres_flexible_server_private" { # private only into UAT and PROD env } resource "azurerm_postgresql_flexible_server_database" "apd_db_flex" { - count = 1 # forced + count = var.env_short != "d" ? 1 : 0 # forced name = var.gpd_db_name server_id = module.postgres_flexible_server_private[0].id @@ -127,7 +127,7 @@ resource "azurerm_postgresql_flexible_server_database" "apd_db_flex" { } resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_connection" { - count = 1 # forced + count = var.env_short != "d" ? 1 : 0 # forced name = "max_connections" server_id = module.postgres_flexible_server_private[0].id @@ -136,7 +136,7 @@ resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_con # Message : FATAL: unsupported startup parameter: extra_float_digits resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_ignore_startup_parameters" { - count = 1 # forced + count = var.env_short != "d" ? 1 : 0 # forced name = "pgbouncer.ignore_startup_parameters" server_id = module.postgres_flexible_server_private[0].id @@ -144,7 +144,7 @@ resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_ignore_ } resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_min_pool_size" { - count = 1 # forced + count = var.env_short != "d" ? 1 : 0 # forced name = "pgbouncer.min_pool_size" server_id = module.postgres_flexible_server_private[0].id @@ -153,13 +153,15 @@ resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_min_poo # CDC https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-logical resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_worker_process" { + count = var.env_short != "d" ? 1 : 0 # forced + name = "max_worker_processes" server_id = module.postgres_flexible_server_private[0].id value = var.pgres_flex_params.max_worker_process # var.env_short == "d" ? 16 : 32 } resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_wal_level" { - count = var.pgres_flex_params.wal_level != null ? 1 : 0 + count = var.pgres_flex_params.wal_level != null && var.env_short != "d" ? 1 : 0 # forced ? 1 : 0 name = "wal_level" server_id = module.postgres_flexible_server_private[0].id @@ -167,7 +169,7 @@ resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_wal_lev } resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_shared_preoload_libraries" { - count = var.pgres_flex_params.wal_level != null ? 1 : 0 + count = var.pgres_flex_params.wal_level != null && var.env_short != "d" ? 1 : 0 name = "shared_preload_libraries" server_id = module.postgres_flexible_server_private[0].id diff --git a/src/domains/gps-common/env/weu-dev/terraform.tfvars b/src/domains/gps-common/env/weu-dev/terraform.tfvars index 780aa5e153..8ba7239ed6 100644 --- a/src/domains/gps-common/env/weu-dev/terraform.tfvars +++ b/src/domains/gps-common/env/weu-dev/terraform.tfvars @@ -75,7 +75,7 @@ pgres_flex_params = { enable_private_dns_registration_virtual_endpoint = false max_worker_process = 16 wal_level = "logical" - shared_preoload_libraries = "pg_failover_slots" + shared_preoload_libraries = "pg_failover_slots,pglogical" public_network_access_enabled = true } diff --git a/src/domains/gps-common/env/weu-uat/terraform.tfvars b/src/domains/gps-common/env/weu-uat/terraform.tfvars index 200e0e04bd..722094a55b 100644 --- a/src/domains/gps-common/env/weu-uat/terraform.tfvars +++ b/src/domains/gps-common/env/weu-uat/terraform.tfvars @@ -77,7 +77,7 @@ pgres_flex_params = { enable_private_dns_registration_virtual_endpoint = false max_worker_process = 32 wal_level = "logical" - shared_preoload_libraries = "pg_failover_slots" + shared_preoload_libraries = "pg_failover_slots,pglogical" public_network_access_enabled = false } From b3dfd2733028f183b31689c0678219e5f8f95007 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Thu, 28 Nov 2024 06:53:44 +0100 Subject: [PATCH 48/55] fix --- src/domains/gps-app/05_debezium_connect.tf | 1 + src/domains/gps-app/99_variables.tf | 6 ++++++ src/domains/gps-app/env/weu-dev/terraform.tfvars | 6 ++++-- src/domains/gps-app/env/weu-prod/terraform.tfvars | 9 ++++++--- src/domains/gps-app/env/weu-uat/terraform.tfvars | 8 +++++--- src/domains/gps-app/yaml/postgres-connector.yaml | 1 + 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index 3d344812ef..70a65b3ca9 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -87,6 +87,7 @@ locals { postgres_username = data.azurerm_key_vault_secret.pgres_gpd_cdc_login.value postgres_password = data.azurerm_key_vault_secret.pgres_gpd_cdc_pwd.value tasks_max = var.tasks_max + max_threads = var.max_threads }) } diff --git a/src/domains/gps-app/99_variables.tf b/src/domains/gps-app/99_variables.tf index 395f82e481..ae4afee7f5 100644 --- a/src/domains/gps-app/99_variables.tf +++ b/src/domains/gps-app/99_variables.tf @@ -543,3 +543,9 @@ variable "limits_cpu" { description = "Connect Limit CPU" default = "0.5" } + +variable "max_threads" { + type = number + description = "Number of max_threads" + default = 1 +} \ No newline at end of file diff --git a/src/domains/gps-app/env/weu-dev/terraform.tfvars b/src/domains/gps-app/env/weu-dev/terraform.tfvars index b680dfcd8b..df1f11c6bc 100644 --- a/src/domains/gps-app/env/weu-dev/terraform.tfvars +++ b/src/domains/gps-app/env/weu-dev/terraform.tfvars @@ -65,7 +65,7 @@ pgbouncer_enabled = false # WISP-dismantling-cfg create_wisp_converter = true -### debezium kafka conn +### debezium zookeeper_yaml zookeeper_replicas = "1" zookeeper_request_memory = "512Mi" zookeeper_request_cpu = "0.5" @@ -74,11 +74,13 @@ zookeeper_limits_cpu = "0.5" zookeeper_jvm_xms = "512m" zookeeper_jvm_xmx = "512m" zookeeper_storage_size = "100Gi" +### debezium kafka_connect_yaml replicas = 1 request_cpu = 0.5 +limits_cpu = 0.5 request_memory = "512Mi" limits_memory = "512Mi" -limits_cpu = 0.5 postgres_db_name = "apd" tasks_max = "1" container_registry = "pagopadcommonacr.azurecr.io" +max_threads = 1 \ No newline at end of file diff --git a/src/domains/gps-app/env/weu-prod/terraform.tfvars b/src/domains/gps-app/env/weu-prod/terraform.tfvars index fe596ecd93..04e6b312da 100644 --- a/src/domains/gps-app/env/weu-prod/terraform.tfvars +++ b/src/domains/gps-app/env/weu-prod/terraform.tfvars @@ -112,6 +112,7 @@ fn_app_storage_account_info = { advanced_threat_protection_enable = true } + ### debezium kafka conn zookeeper_replicas = 3 zookeeper_request_memory = "512Mi" @@ -121,11 +122,13 @@ zookeeper_limits_cpu = 1 zookeeper_jvm_xms = "512m" zookeeper_jvm_xmx = "1024m" zookeeper_storage_size = "100Gi" -replicas = 3 +### debezium kafka_connect_yaml +replicas = 2 request_cpu = 0.5 +limits_cpu = 2 request_memory = "512Mi" -limits_memory = "1024Mi" -limits_cpu = 1 +limits_memory = "3072Mi" postgres_db_name = "apd" tasks_max = "1" container_registry = "pagopapcommonacr.azurecr.io" +max_threads = 10 \ No newline at end of file diff --git a/src/domains/gps-app/env/weu-uat/terraform.tfvars b/src/domains/gps-app/env/weu-uat/terraform.tfvars index da2f964465..ed3fb6e924 100644 --- a/src/domains/gps-app/env/weu-uat/terraform.tfvars +++ b/src/domains/gps-app/env/weu-uat/terraform.tfvars @@ -65,7 +65,7 @@ pgbouncer_enabled = true # WISP-dismantling-cfg create_wisp_converter = true -### debezium kafka conn +### debezium zookeeper_yaml zookeeper_replicas = 3 zookeeper_request_memory = "512Mi" zookeeper_request_cpu = 0.5 @@ -74,11 +74,13 @@ zookeeper_limits_cpu = 1 zookeeper_jvm_xms = "512m" zookeeper_jvm_xmx = "1024m" zookeeper_storage_size = "100Gi" +### debezium kafka_connect_yaml replicas = 1 request_cpu = 0.5 +limits_cpu = 2 request_memory = "512Mi" -limits_memory = "1024Mi" -limits_cpu = 1 +limits_memory = "3072Mi" postgres_db_name = "apd" tasks_max = "1" container_registry = "pagopaucommonacr.azurecr.io" +max_threads = 10 \ No newline at end of file diff --git a/src/domains/gps-app/yaml/postgres-connector.yaml b/src/domains/gps-app/yaml/postgres-connector.yaml index 54aacf7c1f..cb4e8f4bba 100644 --- a/src/domains/gps-app/yaml/postgres-connector.yaml +++ b/src/domains/gps-app/yaml/postgres-connector.yaml @@ -24,5 +24,6 @@ spec: schema.include.list: "apd" table.include.list: "apd.payment_option,apd.payment_position,apd.transfer" plugin.name: "pgoutput" + snapshot.max.threads: ${max_threads} publication.autocreate.mode: "disabled" # shall be create before # publication.autocreate.mode: "filtered" # create it From 76a8bcd9fc511f947d116966d11f6660c04df2c4 Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Thu, 28 Nov 2024 07:17:52 +0100 Subject: [PATCH 49/55] fix --- src/domains/gps-common/03_postgresql_gpd.tf | 8 ++++---- src/domains/gps-common/99_variables.tf | 7 +++++++ src/domains/gps-common/env/weu-dev/terraform.tfvars | 2 ++ src/domains/gps-common/env/weu-prod/terraform.tfvars | 2 ++ src/domains/gps-common/env/weu-uat/terraform.tfvars | 2 ++ 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/domains/gps-common/03_postgresql_gpd.tf b/src/domains/gps-common/03_postgresql_gpd.tf index 067775d43d..1a971a1b86 100644 --- a/src/domains/gps-common/03_postgresql_gpd.tf +++ b/src/domains/gps-common/03_postgresql_gpd.tf @@ -14,7 +14,7 @@ data "azurerm_key_vault_secret" "pgres_admin_pwd" { } resource "azurerm_resource_group" "flex_data" { - count = 1 # forced + count = 1 # forced ( before exits onliy in UAT and PROD now DEV too) name = format("%s-pgres-flex-rg", local.product) @@ -30,7 +30,7 @@ data "azurerm_resource_group" "data" { module "postgres_flexible_snet" { source = "./.terraform/modules/__v3__/subnet" - count = 1 # forced + count = 1 # forced ( before exits onliy in UAT and PROD now DEV too) name = format("%s-pgres-flexible-snet", local.product) address_prefixes = var.cidr_subnet_pg_flex_dbms @@ -51,7 +51,7 @@ module "postgres_flexible_snet" { } data "azurerm_private_dns_zone" "postgres" { - count = var.env_short != "d" ? 1 : 0 + count = var.env_short != "d" ? 1 : 0 # forced ( before exits onliy in UAT and PROD now DEV too) name = "private.postgres.database.azure.com" resource_group_name = local.vnet_resource_group_name } @@ -59,7 +59,7 @@ data "azurerm_private_dns_zone" "postgres" { # https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-compare-single-server-flexible-server module "postgres_flexible_server_private" { # private only into UAT and PROD env source = "./.terraform/modules/__v3__/postgres_flexible_server" - count = var.env_short != "d" ? 1 : 0 # forced + count = var.env_short != "d" ? 1 : 0 # forced 2 create o delete ( for housekeeping old GPD db) name = format("%s-gpd-pgflex", local.product) diff --git a/src/domains/gps-common/99_variables.tf b/src/domains/gps-common/99_variables.tf index 37127cadfb..f145f9702e 100644 --- a/src/domains/gps-common/99_variables.tf +++ b/src/domains/gps-common/99_variables.tf @@ -461,3 +461,10 @@ variable "location_replica_short" { description = "One of wue, neu" default = "neu" } + + +variable "gpd_cdc_enabled" { + type = bool + description = "Enable CDC for GDP" + default = false +} \ No newline at end of file diff --git a/src/domains/gps-common/env/weu-dev/terraform.tfvars b/src/domains/gps-common/env/weu-dev/terraform.tfvars index 8ba7239ed6..678426daa3 100644 --- a/src/domains/gps-common/env/weu-dev/terraform.tfvars +++ b/src/domains/gps-common/env/weu-dev/terraform.tfvars @@ -142,3 +142,5 @@ gpd_sftp_sa_delete = 2 # GPD Archive account gpd_archive_replication_type = "LRS" + +gpd_cdc_enabled = true diff --git a/src/domains/gps-common/env/weu-prod/terraform.tfvars b/src/domains/gps-common/env/weu-prod/terraform.tfvars index ad682c40c0..18a12e72d9 100644 --- a/src/domains/gps-common/env/weu-prod/terraform.tfvars +++ b/src/domains/gps-common/env/weu-prod/terraform.tfvars @@ -166,3 +166,5 @@ gpd_sftp_sa_delete = 60 # GPD Archive account gpd_archive_replication_type = "GZRS" gpd_sftp_ip_rules = ["37.179.98.148"] + +gpd_cdc_enabled = false diff --git a/src/domains/gps-common/env/weu-uat/terraform.tfvars b/src/domains/gps-common/env/weu-uat/terraform.tfvars index 722094a55b..945f8d8c03 100644 --- a/src/domains/gps-common/env/weu-uat/terraform.tfvars +++ b/src/domains/gps-common/env/weu-uat/terraform.tfvars @@ -136,3 +136,5 @@ gpd_sftp_sa_delete = 7 # GPD Archive account gpd_archive_replication_type = "GRS" + +gpd_cdc_enabled = true From d7f3d712c4f895240d35d6a21ee8662e5756555c Mon Sep 17 00:00:00 2001 From: pasqualespica Date: Thu, 5 Dec 2024 00:28:55 +0100 Subject: [PATCH 50/55] fix --- src/domains/gps-common/03_postgresql_gpd.tf | 75 +++---------------- .../gps-common/03_postgresql_gpd_db.tf | 65 ++++++++++++++-- .../gps-common/03_postgresql_replica.tf | 4 +- src/domains/gps-common/99_locals.tf | 1 + src/domains/gps-common/99_main.tf | 2 +- 5 files changed, 71 insertions(+), 76 deletions(-) diff --git a/src/domains/gps-common/03_postgresql_gpd.tf b/src/domains/gps-common/03_postgresql_gpd.tf index 1a971a1b86..5b48324bdc 100644 --- a/src/domains/gps-common/03_postgresql_gpd.tf +++ b/src/domains/gps-common/03_postgresql_gpd.tf @@ -1,65 +1,10 @@ -# review postgreql: -# naming and resource group -# avaibility zones, backup redundancy - -# KV secrets flex server -data "azurerm_key_vault_secret" "pgres_admin_login" { - name = "pgres-admin-login" - key_vault_id = module.key_vault.id -} - -data "azurerm_key_vault_secret" "pgres_admin_pwd" { - name = "pgres-admin-pwd" - key_vault_id = module.key_vault.id -} - -resource "azurerm_resource_group" "flex_data" { - count = 1 # forced ( before exits onliy in UAT and PROD now DEV too) - - name = format("%s-pgres-flex-rg", local.product) - - location = var.location - tags = var.tags -} -data "azurerm_resource_group" "data" { - name = format("%s-data-rg", local.product) -} - -# Postgres Flexible Server subnet -module "postgres_flexible_snet" { - source = "./.terraform/modules/__v3__/subnet" - - count = 1 # forced ( before exits onliy in UAT and PROD now DEV too) - - name = format("%s-pgres-flexible-snet", local.product) - address_prefixes = var.cidr_subnet_pg_flex_dbms - resource_group_name = local.vnet_resource_group_name - virtual_network_name = local.vnet_name - service_endpoints = ["Microsoft.Storage"] - private_endpoint_network_policies_enabled = false - - delegation = { - name = "delegation" - service_delegation = { - name = "Microsoft.DBforPostgreSQL/flexibleServers" - actions = [ - "Microsoft.Network/virtualNetworks/subnets/join/action", - ] - } - } -} - -data "azurerm_private_dns_zone" "postgres" { - count = var.env_short != "d" ? 1 : 0 # forced ( before exits onliy in UAT and PROD now DEV too) - name = "private.postgres.database.azure.com" - resource_group_name = local.vnet_resource_group_name -} +# NEWGPD-DB : DEPRECATED remove all content file # https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-compare-single-server-flexible-server module "postgres_flexible_server_private" { # private only into UAT and PROD env source = "./.terraform/modules/__v3__/postgres_flexible_server" - count = var.env_short != "d" ? 1 : 0 # forced 2 create o delete ( for housekeeping old GPD db) + count = var.env_short == "p" ? 1 : 0 # forced 2 create o delete ( for housekeeping old GPD db) name = format("%s-gpd-pgflex", local.product) @@ -118,7 +63,7 @@ module "postgres_flexible_server_private" { # private only into UAT and PROD env } resource "azurerm_postgresql_flexible_server_database" "apd_db_flex" { - count = var.env_short != "d" ? 1 : 0 # forced + count = var.env_short == "p" ? 1 : 0 # forced name = var.gpd_db_name server_id = module.postgres_flexible_server_private[0].id @@ -127,7 +72,7 @@ resource "azurerm_postgresql_flexible_server_database" "apd_db_flex" { } resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_connection" { - count = var.env_short != "d" ? 1 : 0 # forced + count = var.env_short == "p" ? 1 : 0 # forced name = "max_connections" server_id = module.postgres_flexible_server_private[0].id @@ -136,7 +81,7 @@ resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_con # Message : FATAL: unsupported startup parameter: extra_float_digits resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_ignore_startup_parameters" { - count = var.env_short != "d" ? 1 : 0 # forced + count = var.env_short == "p" ? 1 : 0 # forced name = "pgbouncer.ignore_startup_parameters" server_id = module.postgres_flexible_server_private[0].id @@ -144,16 +89,16 @@ resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_ignore_ } resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_min_pool_size" { - count = var.env_short != "d" ? 1 : 0 # forced + count = var.env_short == "p" ? 1 : 0 # forced name = "pgbouncer.min_pool_size" server_id = module.postgres_flexible_server_private[0].id - value = var.env_short == "d" ? 1 : 10 + value = 10 } # CDC https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-logical resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_worker_process" { - count = var.env_short != "d" ? 1 : 0 # forced + count = var.env_short == "p" ? 1 : 0 # forced name = "max_worker_processes" server_id = module.postgres_flexible_server_private[0].id @@ -161,7 +106,7 @@ resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_max_wor } resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_wal_level" { - count = var.pgres_flex_params.wal_level != null && var.env_short != "d" ? 1 : 0 # forced ? 1 : 0 + count = var.pgres_flex_params.wal_level != null && var.env_short == "p" ? 1 : 0 # forced ? 1 : 0 name = "wal_level" server_id = module.postgres_flexible_server_private[0].id @@ -169,7 +114,7 @@ resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_wal_lev } resource "azurerm_postgresql_flexible_server_configuration" "apd_db_flex_shared_preoload_libraries" { - count = var.pgres_flex_params.wal_level != null && var.env_short != "d" ? 1 : 0 + count = var.pgres_flex_params.wal_level != null && var.env_short == "p" ? 1 : 0 name = "shared_preload_libraries" server_id = module.postgres_flexible_server_private[0].id diff --git a/src/domains/gps-common/03_postgresql_gpd_db.tf b/src/domains/gps-common/03_postgresql_gpd_db.tf index 8fe147474e..7e28d08b2a 100644 --- a/src/domains/gps-common/03_postgresql_gpd_db.tf +++ b/src/domains/gps-common/03_postgresql_gpd_db.tf @@ -1,10 +1,59 @@ -# REMOVE IT after close MS issue -# Support Request: certificates-do-not-conform-to-algorithm -# ########################################################## -# ########################################################## -# ########################################################## -# ########################################################## +# KV secrets flex server +data "azurerm_key_vault_secret" "pgres_admin_login" { + name = "pgres-admin-login" + key_vault_id = module.key_vault.id +} + +data "azurerm_key_vault_secret" "pgres_admin_pwd" { + name = "pgres-admin-pwd" + key_vault_id = module.key_vault.id +} + +resource "azurerm_resource_group" "flex_data" { + count = 1 # forced ( before exits only in UAT and PROD now DEV too) + + name = format("%s-pgres-flex-rg", local.product) + + location = var.location + tags = var.tags +} + +data "azurerm_resource_group" "data" { + name = format("%s-data-rg", local.product) +} + +# Postgres Flexible Server subnet +module "postgres_flexible_snet" { + source = "./.terraform/modules/__v3__/subnet" + + count = 1 # forced ( before exits only in UAT and PROD now DEV too) + + name = format("%s-pgres-flexible-snet", local.product) + address_prefixes = var.cidr_subnet_pg_flex_dbms + resource_group_name = local.vnet_resource_group_name + virtual_network_name = local.vnet_name + service_endpoints = ["Microsoft.Storage"] + private_endpoint_network_policies_enabled = false + + delegation = { + name = "delegation" + service_delegation = { + name = "Microsoft.DBforPostgreSQL/flexibleServers" + actions = [ + "Microsoft.Network/virtualNetworks/subnets/join/action", + ] + } + } +} + +data "azurerm_private_dns_zone" "postgres" { + count = var.env_short != "d" ? 1 : 0 # forced ( before exits only in UAT and PROD now DEV too) + name = "private.postgres.database.azure.com" + resource_group_name = local.vnet_resource_group_name +} +######## +######## # https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-compare-single-server-flexible-server module "postgres_flexible_server_private_db" { source = "./.terraform/modules/__v3__/postgres_flexible_server" @@ -32,7 +81,7 @@ module "postgres_flexible_server_private_db" { create_mode = null // the update of this argument triggers a replace geo_redundant_backup_enabled = var.pgres_flex_params.geo_redundant_backup_enabled - high_availability_enabled = var.pgres_flex_params.high_availability_enabled + high_availability_enabled = false # var.pgres_flex_params.high_availability_enabled # NEWGPD-DB : DEPRECATED force to false standby_availability_zone = var.pgres_flex_params.standby_availability_zone pgbouncer_enabled = var.pgres_flex_params.pgbouncer_enabled @@ -88,7 +137,7 @@ resource "azurerm_postgresql_flexible_server_configuration" "pd_pgbouncer_ignore resource "azurerm_postgresql_flexible_server_configuration" "pg_pgbouncer_min_pool_size" { name = "pgbouncer.min_pool_size" server_id = module.postgres_flexible_server_private_db.id - value = var.env_short == "d" ? 1 : 10 + value = 10 } # CDC https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-logical diff --git a/src/domains/gps-common/03_postgresql_replica.tf b/src/domains/gps-common/03_postgresql_replica.tf index 2856503b4e..9219548604 100644 --- a/src/domains/gps-common/03_postgresql_replica.tf +++ b/src/domains/gps-common/03_postgresql_replica.tf @@ -40,7 +40,7 @@ module "postgresql_gpd_replica_db" { high_availability_enabled = false pgbouncer_enabled = var.pgres_flex_params.pgbouncer_enabled - source_server_id = module.postgres_flexible_server_private[0].id + source_server_id = module.postgres_flexible_server_private[0].id # NEWGPD-DB : DEPRECATED switch to new istance postgres_flexible_server_private_db diagnostic_settings_enabled = false @@ -53,7 +53,7 @@ resource "null_resource" "virtual_endpoint" { count = var.geo_replica_enabled ? 1 : 0 triggers = { rg_name = azurerm_resource_group.flex_data[0].name - primary_server_name = module.postgres_flexible_server_private[0].name + primary_server_name = module.postgres_flexible_server_private[0].name # NEWGPD-DB : DEPRECATED switch to new istance postgres_flexible_server_private_db ve_name = "${local.project}-pgflex-ve" member_name = module.postgresql_gpd_replica_db[0].name } diff --git a/src/domains/gps-common/99_locals.tf b/src/domains/gps-common/99_locals.tf index fcac481c02..f522bcb7c1 100644 --- a/src/domains/gps-common/99_locals.tf +++ b/src/domains/gps-common/99_locals.tf @@ -39,6 +39,7 @@ locals { aks_subnet_name = "${var.prefix}-${var.env_short}-${var.location_short}-${var.env}-aks-snet" azdo_subnet_name = "${local.product}-azdoa-snet" + # NEWGPD-DB : DEPRECATED switch to new istance postgres_flexible_server_private_db gpd_hostname = var.env_short == "p" ? module.postgres_flexible_server_private[0].fqdn : module.postgres_flexible_server_private_db.fqdn gpd_dbmsport = "6432" diff --git a/src/domains/gps-common/99_main.tf b/src/domains/gps-common/99_main.tf index 76569f1b84..eb0908eb50 100644 --- a/src/domains/gps-common/99_main.tf +++ b/src/domains/gps-common/99_main.tf @@ -35,5 +35,5 @@ data "azurerm_subscription" "current" {} data "azurerm_client_config" "current" {} module "__v3__" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=v8.52.0" + source = "git::https://github.com/pagopa/terraform-azurerm-v3?ref=v8.61.1" } \ No newline at end of file From 38fc2d2b341c2126e275248cc6a65666f3d1f000 Mon Sep 17 00:00:00 2001 From: acialini Date: Thu, 5 Dec 2024 15:39:03 +0100 Subject: [PATCH 51/55] [PPANTT-210] feat: Introducing cronjob script to restart if connector is in state failed --- src/domains/gps-app/05_debezium_connect.tf | 25 +++++++++++ .../yaml/debezium-health-checker-cron.yaml | 23 ++++++++++ .../yaml/healthchecker-config-map.yaml | 45 +++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 src/domains/gps-app/yaml/debezium-health-checker-cron.yaml create mode 100644 src/domains/gps-app/yaml/healthchecker-config-map.yaml diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index 70a65b3ca9..8e77486e08 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -90,6 +90,14 @@ locals { max_threads = var.max_threads }) + healthchecker_config_yaml = templatefile("${path.module}/yaml/healthchecker-config-map.yaml", { + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + }) + + debezium-health-checker-cron_yaml = templatefile("${path.module}/yaml/debezium-health-checker-cron.yaml", { + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + }) + } resource "kubectl_manifest" "debezium_role" { @@ -162,3 +170,20 @@ resource "null_resource" "wait_postgres_connector" { interpreter = ["/bin/bash", "-c"] } } + +resource "kubectl_manifest" "healthchecker-config-map" { + depends_on = [ + helm_release.strimzi-kafka-operator + ] + force_conflicts = true + yaml_body = local.healthchecker_config_yaml +} + +resource "kubectl_manifest" "healthchecker-cron" { + depends_on = [ + helm_release.strimzi-kafka-operator, kubectl_manifest.healthchecker-config-map + ] + force_conflicts = true + yaml_body = local.debezium-health-checker-cron_yaml +} + diff --git a/src/domains/gps-app/yaml/debezium-health-checker-cron.yaml b/src/domains/gps-app/yaml/debezium-health-checker-cron.yaml new file mode 100644 index 0000000000..5796817037 --- /dev/null +++ b/src/domains/gps-app/yaml/debezium-health-checker-cron.yaml @@ -0,0 +1,23 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: debezium-health-checker-cron + namespace: ${namespace} +spec: + schedule: "*/5 * * * *" # Runs every 5 minutes + jobTemplate: + spec: + template: + spec: + containers: + - name: debezium-health-checker + image: bitnami/kubectl:latest + command: ["/bin/bash", "/scripts/check_kafka_connect.sh"] + volumeMounts: + - name: script-volume + mountPath: /scripts + restartPolicy: OnFailure + volumes: + - name: script-volume + configMap: + name: health-checker-script diff --git a/src/domains/gps-app/yaml/healthchecker-config-map.yaml b/src/domains/gps-app/yaml/healthchecker-config-map.yaml new file mode 100644 index 0000000000..b722c0d5a0 --- /dev/null +++ b/src/domains/gps-app/yaml/healthchecker-config-map.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: health-checker-script + namespace: ${namespace} +data: + check_kafka_connect.sh: | + #!/bin/bash + + # Kafka Connect API details + STATUS_URL="http://debezium-connect-cluster-connect-api:8083/connectors/debezium-connector-postgres/status" + RESTART_URL="http://debezium-connect-cluster-connect-api:8083/connectors/debezium-connector-postgres/restart" + + # Fetch the status response + STATUS_RESPONSE=$(curl -s -X GET "$STATUS_URL") + + # Extract the connector state + CONNECTOR_STATUS=$(echo "$STATUS_RESPONSE" | grep -o '"connector":{"state":"[^"]*"' | sed 's/"connector":{"state":"//;s/"//') + + # Extract all task states + # TASK_STATUSES=$(echo "$STATUS_RESPONSE" | grep -o '"state":"[^"]*"' | sed 's/"state":"//;s/"//') + + # Initialize FAILED flag + IS_FAILED=0 + + # Check if the connector state is FAILED + if [[ "$CONNECTOR_STATUS" == "FAILED" ]]; then + IS_FAILED=1 + fi + + # Check if any task state is FAILED + # while IFS= read -r TASK_STATE; do + # if [[ "$TASK_STATE" == "FAILED" ]]; then + # IS_FAILED=1 + # break + # fi + # done <<< "$TASK_STATUSES" + + # Take action based on the combined status + if [[ $IS_FAILED -eq 1 ]]; then + echo "One or more statuses are FAILED. Restarting the connector..." + curl -s -X POST "$RESTART_URL" && echo "Restart command issued successfully." + else + echo "All statuses are normal." + fi From 21a8d24000a0e8b24dd73c39f8da20c2ec72ea6b Mon Sep 17 00:00:00 2001 From: acialini Date: Thu, 5 Dec 2024 15:41:21 +0100 Subject: [PATCH 52/55] [PPANTT-210] feat: Introducing cronjob script to restart if connector is in state failed --- src/domains/gps-app/yaml/debezium-health-checker-cron.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/domains/gps-app/yaml/debezium-health-checker-cron.yaml b/src/domains/gps-app/yaml/debezium-health-checker-cron.yaml index 5796817037..e4e6db5c9d 100644 --- a/src/domains/gps-app/yaml/debezium-health-checker-cron.yaml +++ b/src/domains/gps-app/yaml/debezium-health-checker-cron.yaml @@ -5,8 +5,11 @@ metadata: namespace: ${namespace} spec: schedule: "*/5 * * * *" # Runs every 5 minutes + successfulJobsHistoryLimit: 0 + failedJobsHistoryLimit: 1 jobTemplate: spec: + ttlSecondsAfterFinished: 100 template: spec: containers: From 6e5f08ff4df6a6d3c28d0a58b31b81583d7e391a Mon Sep 17 00:00:00 2001 From: acialini Date: Fri, 6 Dec 2024 10:40:48 +0100 Subject: [PATCH 53/55] [PPANTT-210] feat: introducing policies for networking --- .../gps-app/yaml/debezium-ingress.yaml | 31 +++++++++++++++++++ .../gps-app/yaml/debezium-network-policy.yaml | 28 +++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 src/domains/gps-app/yaml/debezium-ingress.yaml create mode 100644 src/domains/gps-app/yaml/debezium-network-policy.yaml diff --git a/src/domains/gps-app/yaml/debezium-ingress.yaml b/src/domains/gps-app/yaml/debezium-ingress.yaml new file mode 100644 index 0000000000..022e67b2f8 --- /dev/null +++ b/src/domains/gps-app/yaml/debezium-ingress.yaml @@ -0,0 +1,31 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-body-size: 1m + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/use-regex: "true" + creationTimestamp: "2024-12-06T08:55:55Z" + generation: 1 + labels: + ingress: debezium + name: debezium-ingress + namespace: gps +spec: + ingressClassName: nginx + rules: + - host: weudev.gps.internal.dev.platform.pagopa.it + http: + paths: + - backend: + service: + name: debezium-connect-cluster-connect-api + port: + number: 8083 + path: /debezium/(.*) + pathType: ImplementationSpecific + tls: + - hosts: + - weudev.gps.internal.dev.platform.pagopa.it + secretName: weudev-gps-internal-dev-platform-pagopa-it diff --git a/src/domains/gps-app/yaml/debezium-network-policy.yaml b/src/domains/gps-app/yaml/debezium-network-policy.yaml new file mode 100644 index 0000000000..16e612d8d9 --- /dev/null +++ b/src/domains/gps-app/yaml/debezium-network-policy.yaml @@ -0,0 +1,28 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: debezium-connect-cluster-network-policy +spec: + ingress: + - from: + - podSelector: + matchLabels: + ingress: debezium + - podSelector: + matchLabels: + strimzi.io/cluster: debezium-connect-cluster + strimzi.io/kind: KafkaConnect + strimzi.io/name: debezium-connect-cluster-connect + - podSelector: + matchLabels: + strimzi.io/kind: cluster-operator + ports: + - port: 8083 + protocol: TCP + podSelector: + matchLabels: + strimzi.io/cluster: debezium-connect-cluster + strimzi.io/kind: KafkaConnect + strimzi.io/name: debezium-connect-cluster-connect + policyTypes: + - Ingress From ded4f7f797586086e9173d6169ab45eb49156e6e Mon Sep 17 00:00:00 2001 From: acialini Date: Fri, 6 Dec 2024 12:33:49 +0100 Subject: [PATCH 54/55] [PPANTT-210] feat: introducing updated script and network policies --- src/domains/gps-app/05_debezium_connect.tf | 30 +++++++++++- .../gps-app/yaml/debezium-ingress.yaml | 10 ++-- .../gps-app/yaml/debezium-network-policy.yaml | 1 + .../yaml/healthchecker-config-map.yaml | 47 +++++++------------ 4 files changed, 51 insertions(+), 37 deletions(-) diff --git a/src/domains/gps-app/05_debezium_connect.tf b/src/domains/gps-app/05_debezium_connect.tf index 8e77486e08..4a64ac2c87 100644 --- a/src/domains/gps-app/05_debezium_connect.tf +++ b/src/domains/gps-app/05_debezium_connect.tf @@ -94,10 +94,20 @@ locals { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name }) - debezium-health-checker-cron_yaml = templatefile("${path.module}/yaml/debezium-health-checker-cron.yaml", { + debezium_health_checker_cron_yaml = templatefile("${path.module}/yaml/debezium-health-checker-cron.yaml", { namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name }) + debezium_network_policy_yaml = templatefile("${path.module}/yaml/debezium-network-policy.yaml", { + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + }) + + debezium_ingress_yaml = templatefile("${path.module}/yaml/debezium-ingress.yaml", { + namespace = "gps" # kubernetes_namespace.namespace.metadata[0].name + host = "${var.location_short}${var.env_short}.gps.internal.${var.env_short}.platform.pagopa.it" + secret = "${var.location_short}${var.env_short}-gps-internal-${var.env_short}-platform-pagopa-it" + }) + } resource "kubectl_manifest" "debezium_role" { @@ -184,6 +194,22 @@ resource "kubectl_manifest" "healthchecker-cron" { helm_release.strimzi-kafka-operator, kubectl_manifest.healthchecker-config-map ] force_conflicts = true - yaml_body = local.debezium-health-checker-cron_yaml + yaml_body = local.debezium_health_checker_cron_yaml +} + +resource "kubectl_manifest" "debezium-ingress" { + depends_on = [ + kubectl_manifest.kafka_connect + ] + force_conflicts = true + yaml_body = local.debezium_ingress_yaml +} + +resource "kubectl_manifest" "debezium-network-policy" { + depends_on = [ + kubectl_manifest.kafka_connect + ] + force_conflicts = true + yaml_body = local.debezium_network_policy_yaml } diff --git a/src/domains/gps-app/yaml/debezium-ingress.yaml b/src/domains/gps-app/yaml/debezium-ingress.yaml index 022e67b2f8..f75e9ef645 100644 --- a/src/domains/gps-app/yaml/debezium-ingress.yaml +++ b/src/domains/gps-app/yaml/debezium-ingress.yaml @@ -6,16 +6,14 @@ metadata: nginx.ingress.kubernetes.io/proxy-body-size: 1m nginx.ingress.kubernetes.io/rewrite-target: /$1 nginx.ingress.kubernetes.io/use-regex: "true" - creationTimestamp: "2024-12-06T08:55:55Z" - generation: 1 labels: ingress: debezium name: debezium-ingress - namespace: gps + namespace: ${namespace} spec: ingressClassName: nginx rules: - - host: weudev.gps.internal.dev.platform.pagopa.it + - host: ${host} http: paths: - backend: @@ -27,5 +25,5 @@ spec: pathType: ImplementationSpecific tls: - hosts: - - weudev.gps.internal.dev.platform.pagopa.it - secretName: weudev-gps-internal-dev-platform-pagopa-it + - ${host} + secretName: ${secret} diff --git a/src/domains/gps-app/yaml/debezium-network-policy.yaml b/src/domains/gps-app/yaml/debezium-network-policy.yaml index 16e612d8d9..3556d0678e 100644 --- a/src/domains/gps-app/yaml/debezium-network-policy.yaml +++ b/src/domains/gps-app/yaml/debezium-network-policy.yaml @@ -2,6 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: debezium-connect-cluster-network-policy + namespace: {namespace} spec: ingress: - from: diff --git a/src/domains/gps-app/yaml/healthchecker-config-map.yaml b/src/domains/gps-app/yaml/healthchecker-config-map.yaml index b722c0d5a0..9add933d42 100644 --- a/src/domains/gps-app/yaml/healthchecker-config-map.yaml +++ b/src/domains/gps-app/yaml/healthchecker-config-map.yaml @@ -6,40 +6,29 @@ metadata: data: check_kafka_connect.sh: | #!/bin/bash - - # Kafka Connect API details STATUS_URL="http://debezium-connect-cluster-connect-api:8083/connectors/debezium-connector-postgres/status" - RESTART_URL="http://debezium-connect-cluster-connect-api:8083/connectors/debezium-connector-postgres/restart" + CONNECTOR_RESTART_URL="http://debezium-connect-cluster-connect-api:8083/connectors/debezium-connector-postgres/restart" + TASK_RESTART_URL="http://debezium-connect-cluster-connect-api:8083/connectors/debezium-connector-postgres/tasks" - # Fetch the status response STATUS_RESPONSE=$(curl -s -X GET "$STATUS_URL") - # Extract the connector state CONNECTOR_STATUS=$(echo "$STATUS_RESPONSE" | grep -o '"connector":{"state":"[^"]*"' | sed 's/"connector":{"state":"//;s/"//') - # Extract all task states - # TASK_STATUSES=$(echo "$STATUS_RESPONSE" | grep -o '"state":"[^"]*"' | sed 's/"state":"//;s/"//') - - # Initialize FAILED flag - IS_FAILED=0 - - # Check if the connector state is FAILED - if [[ "$CONNECTOR_STATUS" == "FAILED" ]]; then - IS_FAILED=1 - fi - - # Check if any task state is FAILED - # while IFS= read -r TASK_STATE; do - # if [[ "$TASK_STATE" == "FAILED" ]]; then - # IS_FAILED=1 - # break - # fi - # done <<< "$TASK_STATUSES" - - # Take action based on the combined status - if [[ $IS_FAILED -eq 1 ]]; then - echo "One or more statuses are FAILED. Restarting the connector..." - curl -s -X POST "$RESTART_URL" && echo "Restart command issued successfully." + if [[ "$CONNECTOR_STATUS" != "RUNNING" ]]; then + echo "Connector is not running (state: $CONNECTOR_STATUS). Restarting..." + curl -s -X POST "$CONNECTOR_RESTART_URL" && echo "Connector restart command issued." else - echo "All statuses are normal." + echo "Connector is running normally." fi + + TASKS=$(echo "$STATUS_RESPONSE" | grep -o '"id":[0-9]*,"state":"[^"]*"' | sed 's/"id"://;s/"state":"//;s/,/ /') + + echo "$TASKS" | while read -r TASK_ID TASK_STATE; do + if [[ "$TASK_STATE" != "RUNNING" ]]; then + echo "Task $TASK_ID is not running (state: $TASK_STATE). Restarting..." + TASK_RESTART_API="${TASK_RESTART_URL}/${TASK_ID}/restart" + curl -s -X POST "$TASK_RESTART_API" && echo "Task $TASK_ID restart command issued." + else + echo "Task $TASK_ID is running normally." + fi + done From 9187e2841a00f8f81938adb7ba9a9875364ac2ca Mon Sep 17 00:00:00 2001 From: acialini Date: Mon, 9 Dec 2024 09:49:40 +0100 Subject: [PATCH 55/55] [PPANTT-210] feat: Debezium GPD API --- src/domains/gps-app/04_apim_gpd_debezium.tf | 76 +++++++++ .../api/debezium-api/v1/_base_policy.xml | 28 ++++ .../api/debezium-api/v1/_openapi.json.tpl | 156 ++++++++++++++++++ .../api_product/debezium-api/_base_policy.xml | 26 +++ .../gps-app/yaml/debezium-ingress.yaml | 2 +- 5 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 src/domains/gps-app/04_apim_gpd_debezium.tf create mode 100644 src/domains/gps-app/api/debezium-api/v1/_base_policy.xml create mode 100644 src/domains/gps-app/api/debezium-api/v1/_openapi.json.tpl create mode 100644 src/domains/gps-app/api_product/debezium-api/_base_policy.xml diff --git a/src/domains/gps-app/04_apim_gpd_debezium.tf b/src/domains/gps-app/04_apim_gpd_debezium.tf new file mode 100644 index 0000000000..6134e34ad9 --- /dev/null +++ b/src/domains/gps-app/04_apim_gpd_debezium.tf @@ -0,0 +1,76 @@ +#################### +## Local variables # +#################### + +locals { + apim_gpd_debezium_api = { + published = false + subscription_required = true + approval_required = false + subscriptions_limit = 1000 + service_url = format("https://%s/debezium-gpd", local.gps_hostname) + } +} + +############## +## Products ## +############## + +module "apim_gpd_debezium_product" { + source = "./.terraform/modules/__v3__/api_management_product" + + product_id = "product-gpd-debezium" + display_name = "GPD Debezium API pagoPA" + description = "Prodotto GPD Debezium API" + + api_management_name = local.pagopa_apim_name + resource_group_name = local.pagopa_apim_rg + + published = local.apim_gpd_debezium_api.published + subscription_required = local.apim_gpd_debezium_api.subscription_required + approval_required = local.apim_gpd_debezium_api.approval_required + subscriptions_limit = local.apim_gpd_debezium_api.subscriptions_limit + + policy_xml = file("./api_product/debezium-api/_base_policy.xml") +} + +############## +## API ## +############## + +resource "azurerm_api_management_api_version_set" "api_gpd_debezium_api" { + + name = format("%s-api-gpd-debezium-api", var.env_short) + api_management_name = local.pagopa_apim_name + resource_group_name = local.pagopa_apim_rg + display_name = "GPD Debezium API" + versioning_scheme = "Segment" +} + + +module "apim_api_gpd_debezium_api" { + source = "./.terraform/modules/__v3__/api_management_api" + + name = format("%s-api-gpd-debezium-api", var.env_short) + api_management_name = local.pagopa_apim_name + resource_group_name = local.pagopa_apim_rg + product_ids = [module.apim_gpd_debezium_product.product_id, module.apim_gpd_debezium_product.product_id] + subscription_required = local.apim_gpd_debezium_api.subscription_required + api_version = "v1" + version_set_id = azurerm_api_management_api_version_set.api_gpd_debezium_api.id + service_url = format("https://%s", module.reporting_analysis_function.default_hostname) + + description = "Api GPD Debezium" + display_name = "GPDDebezium API pagoPA" + path = "gpd-debezium/api" + protocols = ["https"] + + content_format = "openapi" + content_value = templatefile("./api/debezium-api/v1/_openapi.json.tpl", { + host = local.apim_hostname + }) + + xml_content = templatefile("./api/debezium-api/v1/_base_policy.xml", { + origin = format("https://%s.%s.%s", var.cname_record_name, var.apim_dns_zone_prefix, var.external_domain) + }) +} diff --git a/src/domains/gps-app/api/debezium-api/v1/_base_policy.xml b/src/domains/gps-app/api/debezium-api/v1/_base_policy.xml new file mode 100644 index 0000000000..f9edd06caf --- /dev/null +++ b/src/domains/gps-app/api/debezium-api/v1/_base_policy.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + diff --git a/src/domains/gps-app/api/debezium-api/v1/_openapi.json.tpl b/src/domains/gps-app/api/debezium-api/v1/_openapi.json.tpl new file mode 100644 index 0000000000..bd0a560a71 --- /dev/null +++ b/src/domains/gps-app/api/debezium-api/v1/_openapi.json.tpl @@ -0,0 +1,156 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Debezium API - GPD", + "version": "1.0.0" + }, + "servers": [ + { + "url": "${host}" + } + ], + "paths": { + "/connectors": { + "get": { + "tags": [ + "Get Connectors List" + ], + "summary": "getConnectors", + "parameters": [], + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + } + } + }, + "400": { + "description": "Error response", + "content": { + "application/json": {} + } + } + } + } + }, + "/connectors/{connectorId}/status": { + "get": { + "tags": [ + "Get Detail on Connector Status" + ], + "summary": "getConnectorStatus", + "parameters": [ + { + "name": "connectorId", + "in": "path", + "schema": { + "type": "string" + }, + "required": true, + "example": "debezium-connector-postgres" + } + ], + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": {} + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": {} + } + } + } + } + }, + "/connectors/{connectorId}/restart": { + "post": { + "tags": [ + "Restart Connector" + ], + "summary": "restartConnector", + "parameters": [ + { + "name": "connectorId", + "in": "path", + "schema": { + "type": "string" + }, + "required": true, + "example": "debezium-connector-postgres" + } + ], + "responses": { + "204": { + "description": "Successful response", + "content": { + "application/json": {} + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": {} + } + } + } + } + }, + "/connectors/{connectorId}/tasks/{taskId}/restart": { + "post": { + "tags": [ + "Restart Task " + ], + "summary": "restartTask", + "parameters": [ + { + "name": "connectorId", + "in": "path", + "schema": { + "type": "string" + }, + "required": true, + "example": "debezium-connector-postgres" + }, + { + "name": "taskId", + "in": "path", + "schema": { + "type": "string" + }, + "required": true, + "example": "0" + } + ], + "responses": { + "204": { + "description": "Successful response", + "content": { + "application/json": {} + } + }, + "404": { + "description": "Not Found", + "content": { + "application/json": {} + } + } + } + } + } + }, + "components": { + "securitySchemes": { + "ApiKey": { + "type": "apiKey", + "description": "The API key to access this function app.", + "name": "Ocp-Apim-Subscription-Key", + "in": "header" + } + } + } +} diff --git a/src/domains/gps-app/api_product/debezium-api/_base_policy.xml b/src/domains/gps-app/api_product/debezium-api/_base_policy.xml new file mode 100644 index 0000000000..ce1df461e7 --- /dev/null +++ b/src/domains/gps-app/api_product/debezium-api/_base_policy.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + diff --git a/src/domains/gps-app/yaml/debezium-ingress.yaml b/src/domains/gps-app/yaml/debezium-ingress.yaml index f75e9ef645..2ee911e76f 100644 --- a/src/domains/gps-app/yaml/debezium-ingress.yaml +++ b/src/domains/gps-app/yaml/debezium-ingress.yaml @@ -21,7 +21,7 @@ spec: name: debezium-connect-cluster-connect-api port: number: 8083 - path: /debezium/(.*) + path: /debezium-gpd/(.*) pathType: ImplementationSpecific tls: - hosts: