fix: paginate aidants connect api results

src/transformer/cli/action/transformation.respository.ts

@@ -14,11 +14,10 @@ import {
   fingerprintsFromLieuxMediationNumeriqueApi,
   saveOutputsInFiles,
   saveFingerprintsInFile,
-  localisationByGeocode,
-  LocalisationByGeo
+  localisationByGeocode
 } from '../../data';
 import { findCommune } from '../../fields';
-import { DataSource, LieuxMediationNumeriqueMatching } from '../../input';
+import { LieuxMediationNumeriqueMatching } from '../../input';
 import { TransformationRepository } from '../../repositories';
 import { diffSinceLastTransform, Fingerprint } from '../diff-since-last-transform';
 import { TransformerOptions } from '../transformer-options';
@@ -40,6 +39,7 @@ export const transformationRespository = async (transformerOptions: TransformerO
     findCommune: findCommune(await communeFromGeoApi()),
     isInQpv: isInQpv(await qpvFromDataGouv()),
     isInZrr: isInZrr(await zrrFromEquipementsSportsGouvApi()),
+    geocode: localisationByGeocode,
     fingerprints,
     saveErrors: writeErrorsInFiles(transformerOptions),
     saveOutputs: useFile
@@ -48,8 +48,6 @@ export const transformationRespository = async (transformerOptions: TransformerO
     diffSinceLastTransform: diffSinceLastTransform(idKey, fingerprints),
     saveFingerprints: useFile
       ? saveFingerprintsInFile(idKey, fingerprints, transformerOptions)
-      : saveFingerprintsWithLieuxMediationNumeriqueApi(idKey, transformerOptions),
-    findLocalisation: async (source: DataSource): Promise<LocalisationByGeo | undefined> =>
-      localisationByGeocode(source, config)
+      : saveFingerprintsWithLieuxMediationNumeriqueApi(idKey, transformerOptions)
   };
 };

src/transformer/cli/action/transformer.action.ts

@@ -1,12 +1,7 @@
 import { LieuMediationNumerique } from '@gouvfr-anct/lieux-de-mediation-numerique';
 import { createHash } from 'crypto';
-import {
-  LocalisationByGeo,
-  sourceATransformer,
-  sourcesFromCartographieNationaleApi,
-  updateSourceWithCartographieNationaleApi
-} from '../../data';
-import { DataSource, LieuMediationNumeriqueWithLocalisation, toLieuxMediationNumerique, validValuesOnly } from '../../input';
+import { sourceATransformer, sourcesFromCartographieNationaleApi, updateSourceWithCartographieNationaleApi } from '../../data';
+import { DataSource, toLieuxMediationNumerique, validValuesOnly } from '../../input';
 import { Report } from '../../report';
 import { TransformationRepository } from '../../repositories';
 import { canTransform, DiffSinceLastTransform } from '../diff-since-last-transform';
@@ -46,32 +41,14 @@ export const transformerAction = async (transformerOptions: TransformerOptions):
   const repository: TransformationRepository = await transformationRespository(transformerOptions);
 
   const diffSinceLastTransform: DiffSinceLastTransform = repository.diffSinceLastTransform(sourceItems);
-  const lieux: DataSource[] = lieuxToTransform(sourceItems, diffSinceLastTransform);
 
   if (nothingToTransform(diffSinceLastTransform)) return;
 
-  const lieuxDeMediationNumerique: LieuMediationNumerique[] = await Promise.all(
-    lieux
-      .map(flatten as (lieu: DataSource) => DataSource)
-      .map(async (dataSource: DataSource, index: number): Promise<LieuMediationNumerique | undefined> => {
-        let lieuDeMediationNumeriqueWithLocalisation: LieuMediationNumeriqueWithLocalisation = toLieuxMediationNumerique(
-          repository,
-          transformerOptions.sourceName,
-          REPORT
-        )(dataSource, index);
-        if (lieuDeMediationNumeriqueWithLocalisation.hasLocalisation === false) {
-          const localisation: LocalisationByGeo | undefined = await repository.findLocalisation(dataSource);
-          lieuDeMediationNumeriqueWithLocalisation = toLieuxMediationNumerique(
-            repository,
-            transformerOptions.sourceName,
-            REPORT
-          )(dataSource, index, localisation);
-        }
-        return lieuDeMediationNumeriqueWithLocalisation.lieuMediationNumerique;
-      })
-  ).then((resolvedLieuxMediationNumerique: (LieuMediationNumerique | undefined)[]): LieuMediationNumerique[] =>
-    resolvedLieuxMediationNumerique.filter(validValuesOnly)
-  );
+  const lieux: DataSource[] = lieuxToTransform(sourceItems, diffSinceLastTransform);
+
+  const lieuxDeMediationNumerique: LieuMediationNumerique[] = (
+    await Promise.all(lieux.map(flatten).map(toLieuxMediationNumerique(repository, transformerOptions.sourceName, REPORT)))
+  ).filter(validValuesOnly);
 
   /* eslint-disable-next-line no-console */
   diffSinceLastTransform != null && console.log('Nouveaux lieux à ajouter :', diffSinceLastTransform.toUpsert.length);

src/transformer/cli/diff-since-last-transform.ts

@@ -62,12 +62,12 @@ const toItemId =
 const idsToDelete = (fingerprints: Fingerprint[], sourceItem: DataSource[], idKey: string): FingerprintToDelete[] =>
   findDeletedIds(fingerprints, sourceItem.map(toItemId(idKey)));
 
-const findItemsToTransform = (sourceItem: DataSource[], fingerprints: Fingerprint[], idKey: string): DiffSinceLastTransform =>
+const findItemsToTransform = (sourceItems: DataSource[], fingerprints: Fingerprint[], idKey: string): DiffSinceLastTransform =>
   idKey === ''
     ? DIFF_WITHOUT_ID
     : {
-        toUpsert: idsToUpsert(sourceItem, idKey, fingerprints),
-        toDelete: idsToDelete(fingerprints, sourceItem, idKey)
+        toUpsert: idsToUpsert(sourceItems, idKey, fingerprints),
+        toDelete: idsToDelete(fingerprints, sourceItems, idKey)
       };
 
 export const diffSinceLastTransform =

src/transformer/data/localisation/localisation-from-geo.ts

@@ -1,36 +1,19 @@
+import { Adresse, Localisation } from '@gouvfr-anct/lieux-de-mediation-numerique';
 import axios, { AxiosResponse } from 'axios';
-import { Colonne, DataSource, Dissociation, Jonction, LieuxMediationNumeriqueMatching } from '../../input';
+import { NO_LOCALISATION } from '../../fields';
 
-export type LocalisationByGeo = {
-  latitude: string;
-  longitude: string;
-};
-
-const isColonne = (colonneToTest: Partial<Colonne> & Partial<Dissociation>): colonneToTest is Colonne =>
-  colonneToTest.colonne != null;
+const isValid = (response: AxiosResponse): boolean =>
+  response.data.features[0]?.geometry?.coordinates != null && response.data.features[0].properties.score > 0.7;
 
-const extractColonneValue = (source: DataSource, field: Colonne | (Jonction & Partial<Colonne>)): string =>
-  isColonne(field) ? source[field.colonne]?.toString() ?? '' : '';
+const toLocalisation = (response: AxiosResponse): Localisation =>
+  Localisation({
+    latitude: response.data.features[0].geometry.coordinates[1],
+    longitude: response.data.features[0].geometry.coordinates[0]
+  });
 
-export const localisationByGeocode = async (
-  source: DataSource,
-  matching: LieuxMediationNumeriqueMatching
-): Promise<LocalisationByGeo | undefined> => {
-  const adresse: string = extractColonneValue(source, matching.adresse)
-    .replace(/\s*\(.*?\)\s*/gu, ' ')
-    .replace(/^(?<adresse>.*),.*$/u, '$<adresse>')
-    .replace(/\s+$/u, '')
-    .replace(/\s/gu, '+');
-  const codePostal: string = extractColonneValue(source, matching.code_postal);
-  const commune: string = extractColonneValue(source, matching.commune);
+export const localisationByGeocode = (adresse: Adresse) => async (): Promise<Localisation> => {
   const response: AxiosResponse = await axios.get(
-    `https://wxs.ign.fr/essentiels/geoportail/geocodage/rest/0.1/search?q=${adresse}&postcode=${codePostal}&city=${commune}`
+    `https://wxs.ign.fr/essentiels/geoportail/geocodage/rest/0.1/search?q=${adresse.voie}&postcode=${adresse.code_postal}&city=${adresse.commune}`
   );
-  let ignReponse: string[] = [];
-  if (response.data.features?.[0]?.geometry != null && response.data.features[0].properties.score > 0.7)
-    ignReponse = response.data.features[0].geometry?.coordinates;
-  return {
-    latitude: ignReponse[1]?.toString() ?? '',
-    longitude: ignReponse[0]?.toString() ?? ''
-  };
+  return isValid(response) ? toLocalisation(response) : NO_LOCALISATION;
 };

src/transformer/data/source/api-or-csv.ts

@@ -29,28 +29,25 @@ const streamPromise = async (
   notJson: boolean,
   chunks: Uint8Array[],
   response: AxiosResponse,
-  key?: string,
   encoding?: string,
   delimiter?: string
-): Promise<string> =>
-  new Promise<string>(
-    (resolve: (promesseValue: PromiseLike<string> | string) => void, reject: (reason?: Error) => void): void => {
+): Promise<object> =>
+  new Promise<object>(
+    (resolve: (promesseValue: PromiseLike<object> | object) => void, reject: (reason?: Error) => void): void => {
       response.data.on('end', async (): Promise<void> => {
         resolve(
-          JSON.stringify(
-            response.headers['content-type'] === 'text/csv' || notJson
-              ? await csv({ delimiter: defaultIfUndefined(delimiter, ',') }).fromString(
-                  iconv.decode(Buffer.concat(chunks), defaultIfUndefined(encoding, 'utf8'))
-                )
-              : fromJson(JSON.parse(Buffer.concat(chunks).toString()), key)
-          )
+          response.headers['content-type'] === 'text/csv' || notJson
+            ? await csv({ delimiter: defaultIfUndefined(delimiter, ',') }).fromString(
+                iconv.decode(Buffer.concat(chunks), defaultIfUndefined(encoding, 'utf8'))
+              )
+            : JSON.parse(Buffer.concat(chunks).toString())
         );
       });
       response.data.on('error', reject);
     }
   );
 
-const streamFromAPI = async (response: AxiosResponse, key?: string, encoding?: string, delimiter?: string): Promise<string> => {
+const streamFromAPI = async (response: AxiosResponse, encoding?: string, delimiter?: string): Promise<object> => {
   const chunks: Uint8Array[] = [];
 
   response.data.on('data', (chunk: Uint8Array): number => chunks.push(chunk));
@@ -62,14 +59,27 @@ const streamFromAPI = async (response: AxiosResponse, key?: string, encoding?: s
     notJson = !inputIsJson(response);
   }
 
-  return streamPromise(notJson, chunks, response, key, encoding, delimiter);
+  return streamPromise(notJson, chunks, response, encoding, delimiter);
 };
 
-const fetchFrom = async ([source, key]: string[], encoding?: string, delimiter?: string): Promise<string> =>
-  streamFromAPI(await axios.get(source ?? '', { responseType: 'stream' }), key, encoding, delimiter);
+const fetchFrom = async ([source, key]: string[], encoding?: string, delimiter?: string): Promise<string[]> => {
+  const response: { next?: string } = await streamFromAPI(
+    await axios.get(source ?? '', { responseType: 'stream' }),
+    encoding,
+    delimiter
+  );
+
+  const data: string[] = fromJson(response, key);
+
+  return response.next == null
+    ? data
+    : [...data, ...(await fetchFrom(`${response.next}@${key}`.split('@'), encoding, delimiter))];
+};
 
 const readFrom = async ([source, key]: string[]): Promise<string> =>
   JSON.stringify(fromJson(JSON.parse(await fs.promises.readFile(source ?? '', 'utf-8')), key));
 
 export const sourceATransformer = async ({ source, encoding = '', delimiter = '' }: SourceSettings): Promise<string> =>
-  source.startsWith('http') ? fetchFrom(source.split('@'), encoding, delimiter) : readFrom(source.split('@'));
+  source.startsWith('http')
+    ? JSON.stringify(await fetchFrom(source.split('@'), encoding, delimiter))
+    : readFrom(source.split('@'));

src/transformer/fields/index.ts

@@ -7,7 +7,7 @@ export * from './horaires/horaires.field';
 export * from './id/id.field';
 export * from './labels-autres';
 export * from './labels-nationaux/labels-nationaux.field';
-export * from './localisation/localisation.field';
+export * from './localisation';
 export * from './modalites-accompagnement/modalites-accompagnement.field';
 export * from './nom/nom.field';
 export * from './pivot/pivot.field';

src/transformer/fields/localisation/geocode.ts

@@ -0,0 +1,3 @@
+import { Adresse, Localisation } from '@gouvfr-anct/lieux-de-mediation-numerique';
+
+export type Geocode = (address: Adresse) => () => Promise<Localisation>;

src/transformer/fields/localisation/index.ts

@@ -0,0 +1,2 @@
+export * from './geocode';
+export * from './localisation.field';