Skip to content

Commit

Permalink
feat: loire atlantique data config (#41)
Browse files Browse the repository at this point in the history
* feat: loire atlantique data config

* fix: modify regex for only dash in email to match with any numbers of dash in

* fix: order of clean operation
  • Loading branch information
abelkhay authored Mar 9, 2023
1 parent ffdac3d commit 296768f
Show file tree
Hide file tree
Showing 8 changed files with 254 additions and 9 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/nightly-publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ jobs:
hub-lo,
mulhouse,
data-inclusion,
paca
paca,
loire-atlantique
]
runs-on: ubuntu-latest
environment: production
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ jobs:
hub-lo,
mulhouse,
data-inclusion,
paca
paca,
loire-atlantique
]
runs-on: ubuntu-latest
environment: production
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/validate.yml
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,8 @@ jobs:
hub-lo,
mulhouse,
data-inclusion,
paca
paca,
loire-atlantique
]
runs-on: ubuntu-latest
environment: demo
Expand Down
196 changes: 196 additions & 0 deletions assets/input/loire-atlantique/loire-atlantique.config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
{
"id": {
"colonne": "recordid"
},
"nom": {
"colonne": "fields.nom_lieu"
},
"code_postal": {
"colonne": "fields.code_postal"
},
"code_insee": {
"colonne": "fields.code_insee"
},
"commune": {
"colonne": "fields.commune"
},
"adresse": {
"colonne": "fields.adresse"
},
"latitude": {
"colonne": "fields.location.0"
},
"longitude": {
"colonne": "fields.location.1"
},
"telephone": {
"colonne": "fields.telephone"
},
"courriel": {
"colonne": "fields.mail_porteur"
},
"site_web": {
"colonne": "fields.site_web"
},
"presentation_detail": {
"colonne": "fields.descriptif"
},
"date_maj": {
"colonne": "fields.date_modif"
},
"conditions_acces": [
{
"colonnes": ["fields.tarifs", "fields.gratuit"],
"termes": ["gratuit", "oui"],
"cible": "Gratuit : Je peux accéder gratuitement au lieu et à ses services"
},
{
"colonnes": ["fields.gratuit"],
"termes": ["non"],
"cible": "Payant : L'accès au lieu et/ou à ses services est payant"
},
{
"colonnes": ["fields.tarifs"],
"termes": ["adhésion"],
"cible": "Adhésion : L'accès au lieu et/ou à ses services nécessite d'y adhérer"
},
{
"colonnes": ["fields.pass_numerique"],
"termes": ["chèque aptic"],
"cible": "Accepte le Pass numérique : Il est possible d'utiliser un Pass numérique pour accéder au lieu"
}
],
"publics_accueillis": [
{
"colonnes": ["fields.public", "public_precisions"],
"termes": ["tout public", "demandeurs d'emploi", "chefs d'entreprise", "salariés", "adultes"],
"cible": "Adultes"
},

{
"colonnes": ["fields.public"],
"termes": ["tout public"],
"cible": "Familles/enfants"
},
{
"colonnes": ["fields.public", "public_precisions"],
"termes": ["tout public", "demandeurs d'emploi", "jeunes", "16-25 ans", "11-17 ans", "chefs d'entreprise", "salariés"],
"cible": "Jeunes (16-26 ans)"
},
{
"colonnes": ["fields.public", "public_precisions"],
"termes": ["tout public", "demandeurs d'emploi", "chefs d'entreprise", "salariés"],
"cible": "Seniors (+ 65 ans)"
},
{
"colonnes": ["fields.accessibilite_pmr"],
"termes": ["oui"],
"cible": "Surdité"
},
{
"colonnes": ["fields.accessibilite_pmr"],
"termes": ["oui"],
"cible": "Handicaps mentaux : déficiences limitant les activités d'une personne"
},
{
"colonnes": ["fields.accessibilite_pmr"],
"termes": ["oui"],
"cible": "Personnes en situation d'illettrisme"
},
{
"colonnes": ["fields.accessibilite_pmr"],
"termes": ["oui"],
"cible": "Public langues étrangères"
},
{
"colonnes": ["fields.public"],
"termes": ["tout public"],
"cible": "Uniquement femmes"
},
{
"colonnes": ["fields.accessibilite_pmr"],
"termes": ["oui"],
"cible": "Handicaps psychiques : troubles psychiatriques donnant lieu à des atteintes comportementales"
},
{
"colonnes": ["fields.accessibilite_pmr"],
"termes": ["oui"],
"cible": "Déficience visuelle"
}
],
"services": [
{
"colonnes": ["fields.activites", "fields.wifi_libelle"],
"termes": ["wi-fi", "wifi"],
"cible": "Accéder à une connexion internet"
},
{
"colonnes": ["fields.activites", "fields.imprimantes_libelle"],
"termes": ["postes informatiques", "poste informatique", "imprimante(s)", "imprimantes"],
"cible": "Accéder à du matériel"
},
{
"colonnes": ["fields.public", "fields.accompagnement_emploi_libelle"],
"termes": ["demandeurs d'emploi", "recherche d'emploi"],
"cible": "Favoriser mon insertion professionnelle"
},
{
"colonnes": ["fields.accompagnement_usages_libelle"],
"termes": ["usages numériques"],
"cible": "Approfondir ma culture numérique"
},
{
"colonnes": ["fields.accompagnement_usages_libelle"],
"termes": ["usages numériques"],
"cible": "Utiliser le numérique au quotidien"
},
{
"colonnes": ["fields.accompagnement_usages_libelle"],
"termes": ["usages numériques"],
"cible": "Promouvoir la citoyenneté numérique"
},
{
"colonnes": ["fields.equipements"],
"termes": ["tablettes", "tablette", "tablette(s)"],
"cible": "Prendre en main un smartphone ou une tablette"
},
{
"colonnes": ["fields.accompagnement_edemarches_libelle", "fields.descriptif"],
"termes": ["démarches administratives", "démarches"],
"cible": "Devenir autonome dans les démarches administratives"
},
{
"colonnes": ["fields.accompagnement_edemarches_libelle", "fields.descriptif"],
"termes": ["démarches administratives", "démarches"],
"cible": "Réaliser des démarches administratives avec un accompagnement"
}
],
"horaires": {
"jours": [
{
"colonne": "fields.horaire_lundi",
"osm": "Mo"
},
{
"colonne": "fields.horaire_mardi",
"osm": "Tu"
},
{
"colonne": "fields.horaire_mercredi",
"osm": "We"
},
{
"colonne": "fields.horaire_jeudi",
"osm": "Th"
},
{
"colonne": "fields.horaire_vendredi",
"osm": "Fr"
},
{
"colonne": "fields.horaire_samedi",
"osm": "Sa"
}
]
}
}
4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
"transformer.mulhouse": "ts-node src/index.ts transformer -n \"Mulhouse\" -t \"Haut-Rhin\" -s \"https://www.data.gouv.fr/fr/datasets/r/e3c4dec5-1c68-4ecf-a420-90924ea61daf\" -c \"./assets/input/mulhouse/mulhouse.config.json\" -o \"./assets/output/mulhouse\"",
"transformer.les-assembleurs": "ts-node src/index.ts transformer -n \"Les Assembleurs\" -t \"Hauts-de-France\" -s \"./assets/input/les-assembleurs/les-assembleurs.json\" -c \"./assets/input/les-assembleurs/les-assembleurs.config.json\" -o \"./assets/output/les-assembleurs\"",
"transformer.paca": "ts-node src/index.ts transformer -n \"Paca\" -t \"Provence-Alpes-Côte d'Azur\" -s \"https://www.data.gouv.fr/fr/datasets/r/8483ce08-a82a-4d44-b227-eb82ed2f9712\" -c \"./assets/input/paca/paca.config.json\" -o \"./assets/output/paca\" -e \"win-1252\" -d \";\"",
"transformer.loire-atlantique": "ts-node src/index.ts transformer -n \"Loire Atlantique\" -t \"Loire-Atlantique\" -s \"https://www.data.gouv.fr/fr/datasets/r/95824460-e707-4db1-a67b-46b4e540d8ac\" -c \"./assets/input/loire-atlantique/loire-atlantique.config.json\" -o \"./assets/output/loire-atlantique\"",
"publier.angers": "ts-node src/index.ts publier -z \"fr:commune:49007\" -m \"./assets/output/angers/publier.json\"",
"publier.conseiller-numerique": "ts-node src/index.ts publier -z \"country:fr\" -m \"./assets/output/conseiller-numerique/publier.json\"",
"publier.data-inclusion": "ts-node src/index.ts publier -z \"country:fr\" -m \"./assets/output/data-inclusion/publier.json\"",
Expand All @@ -70,7 +71,8 @@
"publier.maine-et-loire": "ts-node src/index.ts publier -z \"fr:departement:49\" -m \"./assets/output/maine-et-loire/publier.json\"",
"publier.mulhouse": "ts-node src/index.ts publier -z \"fr:commune:68224\" -m \"./assets/output/mulhouse/publier.json\"",
"publier.les-assembleurs": "ts-node src/index.ts publier -z \"fr:region:32\" -m \"./assets/output/les-assembleurs/publier.json\"",
"publier.paca": "ts-node src/index.ts publier -z \"fr:region:93\" -m \"./assets/output/paca/publier.json\""
"publier.paca": "ts-node src/index.ts publier -z \"fr:region:93\" -m \"./assets/output/paca/publier.json\"",
"publier.loire-atlantique": "ts-node src/index.ts publier -z \"fr:departement:44\" -m \"./assets/output/loire-atlantique/publier.json\""
},
"devDependencies": {
"@commitlint/cli": "^17.1.2",
Expand Down
22 changes: 19 additions & 3 deletions src/transformer/fields/contact/clean-operations.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,20 @@ const fixUppercaseWebsites = (field: string): CleanOperation => ({
fix: (toFix: string): string => toFix.toLowerCase()
});

const fixMissingColonWebsites = (field: string): CleanOperation => ({
name: 'missing colon websites',
selector: /(https?)(\/\/)/u,
field,
fix: (toFix: string): string => toFix.replace(/(https?)(\/\/)/u, '$1:$2')
});

const fixMultipleUrlNotSeparatedWebsites = (field: string): CleanOperation => ({
name: 'missing separator between url',
selector: /(https?:\/\/(?:[^;]+)(?!;))(https?:\/\/)/gu,
field,
fix: (toFix: string): string => toFix.replace(/(https?:\/\/(?:[^;]+)(?!;))(https?:\/\/)/gu, '$1;$2')
});

const fixDuplicateHttpWebsites = (field: string): CleanOperation => ({
name: 'duplicate http websites',
selector: /^https?:\/\/https?:\/\/.*/u,
Expand Down Expand Up @@ -238,26 +252,28 @@ const fixMissingEmailExtension = (field: string): CleanOperation => ({

const removeDashEmail = (field: string): CleanOperation => ({
name: 'dash email',
selector: /^-----$/u,
selector: /^-+$/u,
field
});

const cleanOperationIfAny = (cleanOperator: (colonne: string) => CleanOperation, colonne?: string): CleanOperation[] =>
colonne == null ? [] : [cleanOperator(colonne)];

export const cleanOperations = (matching: LieuxMediationNumeriqueMatching): CleanOperation[] => [
...cleanOperationIfAny(removeDashEmail, matching.courriel?.colonne),
...cleanOperationIfAny(fixDuplicateHttpWebsites, matching.site_web?.colonne),
...cleanOperationIfAny(fixMultipleWebsitesSeparator, matching.site_web?.colonne),
...cleanOperationIfAny(fixUppercaseWebsites, matching.site_web?.colonne),
...cleanOperationIfAny(removeDashEmail, matching.courriel?.colonne),
...cleanOperationIfAny(fixMultipleUrlNotSeparatedWebsites, matching.site_web?.colonne),
...cleanOperationIfAny(removeHttpOnlyWebsites, matching.site_web?.colonne),
...cleanOperationIfAny(removeWebsitesWithAccentedCharacters, matching.site_web?.colonne),
...cleanOperationIfAny(removeMissingExtensionWebsites, matching.site_web?.colonne),
...cleanOperationIfAny(fixDuplicateHttpWebsites, matching.site_web?.colonne),
...cleanOperationIfAny(fixWebsitesWithComaInsteadOfDot, matching.site_web?.colonne),
...cleanOperationIfAny(fixWebsitesWithMissingSlashAfterHttp, matching.site_web?.colonne),
...cleanOperationIfAny(removeWebsitesWithSpaces, matching.site_web?.colonne),
...cleanOperationIfAny(fixMissingHttpWebsites, matching.site_web?.colonne),
...cleanOperationIfAny(fixMissingHttpWebsitesWithMultipleUrl, matching.site_web?.colonne),
...cleanOperationIfAny(fixMissingColonWebsites, matching.site_web?.colonne),
...cleanOperationIfAny(removeStartingByTwoZeroInPhone, matching.telephone?.colonne),
...cleanOperationIfAny(removeNoValidNumbersInPhone, matching.telephone?.colonne),
...cleanOperationIfAny(fixUnexpectedPhoneList, matching.telephone?.colonne),
Expand Down
31 changes: 30 additions & 1 deletion src/transformer/fields/contact/contact.field.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -795,7 +795,6 @@ describe('contact field', (): void => {
{
[EMAIL_FIELD]: '-----',
Téléphone: '3960 (Service 0,06 € / mn + prix appel)'
// 'Site Web': 'http://www.carsat-hdf.fr'
} as DataSource,
matching
);
Expand Down Expand Up @@ -871,4 +870,34 @@ describe('contact field', (): void => {
}
]);
});

it('should add : if missing with https', (): void => {
const contact: Contact = processContact(Report().entry(0))(
{
'Site Web': 'https//www.saintpereenretz.fr/bouger/culture/mediatheque.html'
} as DataSource,
matching
);

expect(contact).toStrictEqual<Contact>(
Contact({
site_web: [Url('https://www.saintpereenretz.fr/bouger/culture/mediatheque.html')]
})
);
});

it('should seperate two url if there is no separator', (): void => {
const contact: Contact = processContact(Report().entry(0))(
{
'Site Web': 'http://www.letoilerie.com/http://marie-et-alphonse.com/'
} as DataSource,
matching
);

expect(contact).toStrictEqual<Contact>(
Contact({
site_web: [Url('http://www.letoilerie.com/'), Url('http://marie-et-alphonse.com/')]
})
);
});
});
1 change: 0 additions & 1 deletion src/transformer/fields/contact/contact.field.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,6 @@ export const processContact =
try {
return toLieuxMediationNumeriqueContact(source, matching);
} catch (error: unknown) {
// console.log('source', source);
error instanceof ModelError && recorder.record(error.key, error.message);
return fixAndRetry(recorder)(source, matching, error);
}
Expand Down

0 comments on commit 296768f

Please sign in to comment.