diff --git a/.github/workflows/nightly-publish.yml b/.github/workflows/nightly-publish.yml index ce5622ff..7933065d 100644 --- a/.github/workflows/nightly-publish.yml +++ b/.github/workflows/nightly-publish.yml @@ -22,7 +22,8 @@ jobs: hub-lo, mulhouse, data-inclusion, - paca + paca, + loire-atlantique ] runs-on: ubuntu-latest environment: production diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ad4e14c..b6420755 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,8 @@ jobs: hub-lo, mulhouse, data-inclusion, - paca + paca, + loire-atlantique ] runs-on: ubuntu-latest environment: production diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 21bcf0be..7d0bd31c 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -65,7 +65,8 @@ jobs: hub-lo, mulhouse, data-inclusion, - paca + paca, + loire-atlantique ] runs-on: ubuntu-latest environment: demo diff --git a/assets/input/loire-atlantique/loire-atlantique.config.json b/assets/input/loire-atlantique/loire-atlantique.config.json new file mode 100644 index 00000000..846dd920 --- /dev/null +++ b/assets/input/loire-atlantique/loire-atlantique.config.json @@ -0,0 +1,196 @@ +{ + "id": { + "colonne": "recordid" + }, + "nom": { + "colonne": "fields.nom_lieu" + }, + "code_postal": { + "colonne": "fields.code_postal" + }, + "code_insee": { + "colonne": "fields.code_insee" + }, + "commune": { + "colonne": "fields.commune" + }, + "adresse": { + "colonne": "fields.adresse" + }, + "latitude": { + "colonne": "fields.location.0" + }, + "longitude": { + "colonne": "fields.location.1" + }, + "telephone": { + "colonne": "fields.telephone" + }, + "courriel": { + "colonne": "fields.mail_porteur" + }, + "site_web": { + "colonne": "fields.site_web" + }, + "presentation_detail": { + "colonne": "fields.descriptif" + }, + "date_maj": { + "colonne": "fields.date_modif" + }, + "conditions_acces": [ + { + "colonnes": ["fields.tarifs", "fields.gratuit"], + "termes": ["gratuit", "oui"], + "cible": "Gratuit : Je peux accéder gratuitement au lieu et à ses services" + }, + { + "colonnes": ["fields.gratuit"], + "termes": ["non"], + "cible": "Payant : L'accès au lieu et/ou à ses services est payant" + }, + { + "colonnes": ["fields.tarifs"], + "termes": ["adhésion"], + "cible": "Adhésion : L'accès au lieu et/ou à ses services nécessite d'y adhérer" + }, + { + "colonnes": ["fields.pass_numerique"], + "termes": ["chèque aptic"], + "cible": "Accepte le Pass numérique : Il est possible d'utiliser un Pass numérique pour accéder au lieu" + } + ], + "publics_accueillis": [ + { + "colonnes": ["fields.public", "public_precisions"], + "termes": ["tout public", "demandeurs d'emploi", "chefs d'entreprise", "salariés", "adultes"], + "cible": "Adultes" + }, + + { + "colonnes": ["fields.public"], + "termes": ["tout public"], + "cible": "Familles/enfants" + }, + { + "colonnes": ["fields.public", "public_precisions"], + "termes": ["tout public", "demandeurs d'emploi", "jeunes", "16-25 ans", "11-17 ans", "chefs d'entreprise", "salariés"], + "cible": "Jeunes (16-26 ans)" + }, + { + "colonnes": ["fields.public", "public_precisions"], + "termes": ["tout public", "demandeurs d'emploi", "chefs d'entreprise", "salariés"], + "cible": "Seniors (+ 65 ans)" + }, + { + "colonnes": ["fields.accessibilite_pmr"], + "termes": ["oui"], + "cible": "Surdité" + }, + { + "colonnes": ["fields.accessibilite_pmr"], + "termes": ["oui"], + "cible": "Handicaps mentaux : déficiences limitant les activités d'une personne" + }, + { + "colonnes": ["fields.accessibilite_pmr"], + "termes": ["oui"], + "cible": "Personnes en situation d'illettrisme" + }, + { + "colonnes": ["fields.accessibilite_pmr"], + "termes": ["oui"], + "cible": "Public langues étrangères" + }, + { + "colonnes": ["fields.public"], + "termes": ["tout public"], + "cible": "Uniquement femmes" + }, + { + "colonnes": ["fields.accessibilite_pmr"], + "termes": ["oui"], + "cible": "Handicaps psychiques : troubles psychiatriques donnant lieu à des atteintes comportementales" + }, + { + "colonnes": ["fields.accessibilite_pmr"], + "termes": ["oui"], + "cible": "Déficience visuelle" + } + ], + "services": [ + { + "colonnes": ["fields.activites", "fields.wifi_libelle"], + "termes": ["wi-fi", "wifi"], + "cible": "Accéder à une connexion internet" + }, + { + "colonnes": ["fields.activites", "fields.imprimantes_libelle"], + "termes": ["postes informatiques", "poste informatique", "imprimante(s)", "imprimantes"], + "cible": "Accéder à du matériel" + }, + { + "colonnes": ["fields.public", "fields.accompagnement_emploi_libelle"], + "termes": ["demandeurs d'emploi", "recherche d'emploi"], + "cible": "Favoriser mon insertion professionnelle" + }, + { + "colonnes": ["fields.accompagnement_usages_libelle"], + "termes": ["usages numériques"], + "cible": "Approfondir ma culture numérique" + }, + { + "colonnes": ["fields.accompagnement_usages_libelle"], + "termes": ["usages numériques"], + "cible": "Utiliser le numérique au quotidien" + }, + { + "colonnes": ["fields.accompagnement_usages_libelle"], + "termes": ["usages numériques"], + "cible": "Promouvoir la citoyenneté numérique" + }, + { + "colonnes": ["fields.equipements"], + "termes": ["tablettes", "tablette", "tablette(s)"], + "cible": "Prendre en main un smartphone ou une tablette" + }, + { + "colonnes": ["fields.accompagnement_edemarches_libelle", "fields.descriptif"], + "termes": ["démarches administratives", "démarches"], + "cible": "Devenir autonome dans les démarches administratives" + }, + { + "colonnes": ["fields.accompagnement_edemarches_libelle", "fields.descriptif"], + "termes": ["démarches administratives", "démarches"], + "cible": "Réaliser des démarches administratives avec un accompagnement" + } + ], + "horaires": { + "jours": [ + { + "colonne": "fields.horaire_lundi", + "osm": "Mo" + }, + { + "colonne": "fields.horaire_mardi", + "osm": "Tu" + }, + { + "colonne": "fields.horaire_mercredi", + "osm": "We" + }, + { + "colonne": "fields.horaire_jeudi", + "osm": "Th" + }, + { + "colonne": "fields.horaire_vendredi", + "osm": "Fr" + }, + { + "colonne": "fields.horaire_samedi", + "osm": "Sa" + } + ] + } +} diff --git a/package.json b/package.json index b5f33afe..d3e8bc4d 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "transformer.mulhouse": "ts-node src/index.ts transformer -n \"Mulhouse\" -t \"Haut-Rhin\" -s \"https://www.data.gouv.fr/fr/datasets/r/e3c4dec5-1c68-4ecf-a420-90924ea61daf\" -c \"./assets/input/mulhouse/mulhouse.config.json\" -o \"./assets/output/mulhouse\"", "transformer.les-assembleurs": "ts-node src/index.ts transformer -n \"Les Assembleurs\" -t \"Hauts-de-France\" -s \"./assets/input/les-assembleurs/les-assembleurs.json\" -c \"./assets/input/les-assembleurs/les-assembleurs.config.json\" -o \"./assets/output/les-assembleurs\"", "transformer.paca": "ts-node src/index.ts transformer -n \"Paca\" -t \"Provence-Alpes-Côte d'Azur\" -s \"https://www.data.gouv.fr/fr/datasets/r/8483ce08-a82a-4d44-b227-eb82ed2f9712\" -c \"./assets/input/paca/paca.config.json\" -o \"./assets/output/paca\" -e \"win-1252\" -d \";\"", + "transformer.loire-atlantique": "ts-node src/index.ts transformer -n \"Loire Atlantique\" -t \"Loire-Atlantique\" -s \"https://www.data.gouv.fr/fr/datasets/r/95824460-e707-4db1-a67b-46b4e540d8ac\" -c \"./assets/input/loire-atlantique/loire-atlantique.config.json\" -o \"./assets/output/loire-atlantique\"", "publier.angers": "ts-node src/index.ts publier -z \"fr:commune:49007\" -m \"./assets/output/angers/publier.json\"", "publier.conseiller-numerique": "ts-node src/index.ts publier -z \"country:fr\" -m \"./assets/output/conseiller-numerique/publier.json\"", "publier.data-inclusion": "ts-node src/index.ts publier -z \"country:fr\" -m \"./assets/output/data-inclusion/publier.json\"", @@ -70,7 +71,8 @@ "publier.maine-et-loire": "ts-node src/index.ts publier -z \"fr:departement:49\" -m \"./assets/output/maine-et-loire/publier.json\"", "publier.mulhouse": "ts-node src/index.ts publier -z \"fr:commune:68224\" -m \"./assets/output/mulhouse/publier.json\"", "publier.les-assembleurs": "ts-node src/index.ts publier -z \"fr:region:32\" -m \"./assets/output/les-assembleurs/publier.json\"", - "publier.paca": "ts-node src/index.ts publier -z \"fr:region:93\" -m \"./assets/output/paca/publier.json\"" + "publier.paca": "ts-node src/index.ts publier -z \"fr:region:93\" -m \"./assets/output/paca/publier.json\"", + "publier.loire-atlantique": "ts-node src/index.ts publier -z \"fr:departement:44\" -m \"./assets/output/loire-atlantique/publier.json\"" }, "devDependencies": { "@commitlint/cli": "^17.1.2", diff --git a/src/transformer/fields/contact/clean-operations.ts b/src/transformer/fields/contact/clean-operations.ts index 7ae89796..83e4d950 100644 --- a/src/transformer/fields/contact/clean-operations.ts +++ b/src/transformer/fields/contact/clean-operations.ts @@ -49,6 +49,20 @@ const fixUppercaseWebsites = (field: string): CleanOperation => ({ fix: (toFix: string): string => toFix.toLowerCase() }); +const fixMissingColonWebsites = (field: string): CleanOperation => ({ + name: 'missing colon websites', + selector: /(https?)(\/\/)/u, + field, + fix: (toFix: string): string => toFix.replace(/(https?)(\/\/)/u, '$1:$2') +}); + +const fixMultipleUrlNotSeparatedWebsites = (field: string): CleanOperation => ({ + name: 'missing separator between url', + selector: /(https?:\/\/(?:[^;]+)(?!;))(https?:\/\/)/gu, + field, + fix: (toFix: string): string => toFix.replace(/(https?:\/\/(?:[^;]+)(?!;))(https?:\/\/)/gu, '$1;$2') +}); + const fixDuplicateHttpWebsites = (field: string): CleanOperation => ({ name: 'duplicate http websites', selector: /^https?:\/\/https?:\/\/.*/u, @@ -238,7 +252,7 @@ const fixMissingEmailExtension = (field: string): CleanOperation => ({ const removeDashEmail = (field: string): CleanOperation => ({ name: 'dash email', - selector: /^-----$/u, + selector: /^-+$/u, field }); @@ -246,18 +260,20 @@ const cleanOperationIfAny = (cleanOperator: (colonne: string) => CleanOperation, colonne == null ? [] : [cleanOperator(colonne)]; export const cleanOperations = (matching: LieuxMediationNumeriqueMatching): CleanOperation[] => [ + ...cleanOperationIfAny(removeDashEmail, matching.courriel?.colonne), + ...cleanOperationIfAny(fixDuplicateHttpWebsites, matching.site_web?.colonne), ...cleanOperationIfAny(fixMultipleWebsitesSeparator, matching.site_web?.colonne), ...cleanOperationIfAny(fixUppercaseWebsites, matching.site_web?.colonne), - ...cleanOperationIfAny(removeDashEmail, matching.courriel?.colonne), + ...cleanOperationIfAny(fixMultipleUrlNotSeparatedWebsites, matching.site_web?.colonne), ...cleanOperationIfAny(removeHttpOnlyWebsites, matching.site_web?.colonne), ...cleanOperationIfAny(removeWebsitesWithAccentedCharacters, matching.site_web?.colonne), ...cleanOperationIfAny(removeMissingExtensionWebsites, matching.site_web?.colonne), - ...cleanOperationIfAny(fixDuplicateHttpWebsites, matching.site_web?.colonne), ...cleanOperationIfAny(fixWebsitesWithComaInsteadOfDot, matching.site_web?.colonne), ...cleanOperationIfAny(fixWebsitesWithMissingSlashAfterHttp, matching.site_web?.colonne), ...cleanOperationIfAny(removeWebsitesWithSpaces, matching.site_web?.colonne), ...cleanOperationIfAny(fixMissingHttpWebsites, matching.site_web?.colonne), ...cleanOperationIfAny(fixMissingHttpWebsitesWithMultipleUrl, matching.site_web?.colonne), + ...cleanOperationIfAny(fixMissingColonWebsites, matching.site_web?.colonne), ...cleanOperationIfAny(removeStartingByTwoZeroInPhone, matching.telephone?.colonne), ...cleanOperationIfAny(removeNoValidNumbersInPhone, matching.telephone?.colonne), ...cleanOperationIfAny(fixUnexpectedPhoneList, matching.telephone?.colonne), diff --git a/src/transformer/fields/contact/contact.field.spec.ts b/src/transformer/fields/contact/contact.field.spec.ts index 07f75464..1af6329f 100644 --- a/src/transformer/fields/contact/contact.field.spec.ts +++ b/src/transformer/fields/contact/contact.field.spec.ts @@ -795,7 +795,6 @@ describe('contact field', (): void => { { [EMAIL_FIELD]: '-----', Téléphone: '3960 (Service 0,06 € / mn + prix appel)' - // 'Site Web': 'http://www.carsat-hdf.fr' } as DataSource, matching ); @@ -871,4 +870,34 @@ describe('contact field', (): void => { } ]); }); + + it('should add : if missing with https', (): void => { + const contact: Contact = processContact(Report().entry(0))( + { + 'Site Web': 'https//www.saintpereenretz.fr/bouger/culture/mediatheque.html' + } as DataSource, + matching + ); + + expect(contact).toStrictEqual( + Contact({ + site_web: [Url('https://www.saintpereenretz.fr/bouger/culture/mediatheque.html')] + }) + ); + }); + + it('should seperate two url if there is no separator', (): void => { + const contact: Contact = processContact(Report().entry(0))( + { + 'Site Web': 'http://www.letoilerie.com/http://marie-et-alphonse.com/' + } as DataSource, + matching + ); + + expect(contact).toStrictEqual( + Contact({ + site_web: [Url('http://www.letoilerie.com/'), Url('http://marie-et-alphonse.com/')] + }) + ); + }); }); diff --git a/src/transformer/fields/contact/contact.field.ts b/src/transformer/fields/contact/contact.field.ts index 7ebd2879..9242bc4e 100644 --- a/src/transformer/fields/contact/contact.field.ts +++ b/src/transformer/fields/contact/contact.field.ts @@ -106,7 +106,6 @@ export const processContact = try { return toLieuxMediationNumeriqueContact(source, matching); } catch (error: unknown) { - // console.log('source', source); error instanceof ModelError && recorder.record(error.key, error.message); return fixAndRetry(recorder)(source, matching, error); }