read geipan csv

package.json

@@ -21,7 +21,7 @@
     "preview": "vite preview --host",
     "chapters": "tsx Chapters.ts",
     "test": "testscript",
-    "test-one": "tsx time/datasource/baseovnifrance/BaseOvniFranceCsv.test.ts",
+    "test-one": "tsx time/datasource/geipan/GeipanCsv.test.ts",
     "udb:build": "webpack"
   },
   "dependencies": {
@@ -33,7 +33,8 @@
     "csv-parser": "^3.0.0",
     "glob-promise": "^6.0.5",
     "image-size": "^1.0.2",
-    "jsdom": "^24.0.0"
+    "jsdom": "^24.0.0",
+    "csv": "^6.3.8"
   },
   "devDependencies": {
     "@javarome/testscript": "^0.10.7",

time/1/4/6/1/06_03_24_1709765760.txt

@@ -0,0 +1,2 @@
+Num cas&D�part.&Ville&Latitude&Longitude&CR Observation&Typ Obs&Date&Heure&Dur�e&Nbre Objets&Type Objet&Couleur&Brillance&Effet visuel&Disp inst&Vitesse&Mouvement/type de vol&Taille&Type Entit�&Nbre Entit�&Action entit�&Effets t�moin&Effet Physique&Nbre T�moins&Enq Off&M�t�o&Ann�e&Atter
+599&62&Arras&50.283&+2.783&Il apparut dans le ciel un objet brillant comme une barre de fer, long et large comme la moiti� de la Lune. Il resta immobile pendant quinze minutes. Puis, soudainement, l'�trange objet commen�a � monter en spirale, puis tourna et virevolta comme un ressort d�tendu de montre et apr�s il disparut dans le ciel.&Visuel : Eloign�&01-11-1461&N.C.&00:15:00&1&Sph�re&M�tallique (argent,)&Brillant&&&&Par � coups (immobilit� puis d�placement)&Moiti� de la pleine Lune (apparente)&&0&, &&&999&n&M�t�o inconnue&1461&n

time/datasource/CaseSource.ts

@@ -1,21 +1,21 @@
-import { HtmlRR0SsgContext, RR0SsgContext } from "../../RR0SsgContext"
+import { RR0SsgContext } from "../../RR0SsgContext"
 
 /**
  * A source for cases.
  */
-export abstract class CaseSource<S> {
+export interface CaseSource<S> {
   /**
    * @param author The datasource author to mention as a source.
    * @param copyright The datasource name/copyright to mention as a source.
    * @protected
    */
-  protected constructor(readonly author: string, readonly copyright: string) {
-  }
+  readonly author: string
+  readonly copyright: string
 
   /**
    * Get cases matching a context, in the datasource native format.
    *
    * @param context
    */
-  abstract getAll(context: RR0SsgContext): Promise<S[]>
+  getAll(context: RR0SsgContext): Promise<S[]>
 }

time/datasource/CsvMapper.ts

@@ -7,7 +7,7 @@ export class CsvMapper<S> implements CaseMapper<RR0SsgContext, S, string> {
   readonly fields = new Set<string>()
 
   constructor(
-    readonly sep = ",", readonly escapeChar = "\"", readonly prefix = "") {
+    readonly sep = ",", readonly escapeStr = "\"", readonly prefix = "") {
   }
 
   readonly fieldMapper = (context: RR0SsgContext, key: string, value: any, sourceTime: Date): string => {
@@ -22,7 +22,7 @@ export class CsvMapper<S> implements CaseMapper<RR0SsgContext, S, string> {
     } else if (Array.isArray(value)) {
       val = value.map((item, i) => this.fieldMapper(context, String(i), item, sourceTime)).join(";")
     } else if (typeof value === "object") {
-      const subMapper = new CsvMapper(this.sep, this.escapeChar, this.prefix + key + ".")
+      const subMapper = new CsvMapper(this.sep, this.escapeStr, this.prefix + key + ".")
       const subValues = subMapper.map(context, value, sourceTime)
       let addSubFields = !isFinite(key)
       if (addSubFields) {
@@ -64,26 +64,49 @@ export class CsvMapper<S> implements CaseMapper<RR0SsgContext, S, string> {
   }
 
   escape(value: string): string {
-    return value.indexOf(this.sep) >= 0 ? this.escapeChar + value + this.escapeChar : value
-  }
-
-  readLine(line: string): S {
-    const c = {}
-    const values = line.split(this.sep)
-    const fields = Array.from(this.fields)
-    for (let i = 0; i < fields.length; i++) {
-      const field = fields[i]
-      c[field] = values[i]
-    }
-    return c as S
+    return this.escapeStr && value.indexOf(this.sep) >= 0 ? this.escapeStr + value + this.escapeStr : value
   }
 
   read(context: RR0SsgContext, data: string): S[] {
-    const lines = data.split("\n")
-    const header = lines.shift()
+    let eol = data.indexOf("\n")
+    const header = data.substring(0, eol)
+    data = data.substring(eol + 1).replaceAll(`""`, "''")
     this.fields.clear()
     const columns = header.split(this.sep)
     columns.forEach(column => this.fields.add(column))
-    return lines.filter(l => l.trim()).map(l => this.readLine(l))
+    const records: S[] = []
+    let regex = new RegExp(`(?:${this.escapeStr}(.*?)${this.escapeStr}(?:${this.sep}|\n))|(?:(.*?)(?:${this.sep}|\n))`,
+      "gs")
+    let values = []
+    let m
+    const fields = Array.from(this.fields)
+    while ((m = regex.exec(data)) !== null) {
+      if (m.index === regex.lastIndex) {  // This is necessary to avoid infinite loops with zero-width matches
+        regex.lastIndex++
+        if (regex.lastIndex > data.length) {
+          break
+        }
+      }
+      m.forEach((match, group) => {
+        let empty = match === this.sep && group
+        if (match !== undefined && group) {
+          const val = empty ? "" : match
+          values.push(val)
+          const c = {}
+          if (values.length === fields.length) {
+            for (let i = 0; i < fields.length; i++) {
+              const field = fields[i]
+              c[field] = values[i]
+            }
+            records.push(c as S)
+            values = []
+            data = data.substring(regex.lastIndex)
+            regex = new RegExp(`(?:${this.escapeStr}(.*?)${this.escapeStr}(?:${this.sep}|\n))|(?:(.*?)(?:${this.sep}|\n))`,
+              "gs")
+          }
+        }
+      })
+    }
+    return records
   }
 }

time/datasource/HttpCaseSource.ts → time/datasource/HttpSource.ts

@@ -1,7 +1,3 @@
-import { CaseSource } from "./CaseSource"
-import { HtmlRR0SsgContext } from "../../RR0SsgContext"
-import { RR0Case } from "../RR0Case"
-
 export class MimeType {
   static readonly csv: string = "text/csv"
   static readonly json: string = "application/json"
@@ -12,11 +8,9 @@ export class MimeType {
 /**
  * A source for cases that can be fetched online.
  */
-export abstract class HttpCaseSource<S> extends CaseSource<S> {
+export class HttpSource {
 
   protected constructor(
-    author: string,
-    copyright: string,
     protected userAgents = [
       "Mozilla/5.0 (Windows NT 10.0; Win64; x64)  AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
       "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36",
@@ -26,7 +20,6 @@ export abstract class HttpCaseSource<S> extends CaseSource<S> {
       "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36"
     ]
   ) {
-    super(author, copyright)
   }
 
   static findParam(str: string, separator: string, param: string): string {
@@ -43,19 +36,19 @@ export abstract class HttpCaseSource<S> extends CaseSource<S> {
     return foundParam.substring(key.length)
   }
 
-  protected randomUA() {
+  randomUA() {
     const randomNumber = Math.floor(Math.random() * this.userAgents.length)
     return this.userAgents[randomNumber]
   }
 
-  protected async fetch<T>(url: string, init: RequestInit): Promise<T> {
+  async fetch<T>(url: string, init: RequestInit): Promise<T> {
     init.headers = Object.assign({"User-Agent": this.randomUA()}, init.headers)
     const response = await fetch(url, init)
     if (response.ok) {
       const accept = init.headers["accept"]
       if (accept) {
         const buffer = await response.arrayBuffer()
-        const charset = HttpCaseSource.findParam(accept, ";", "charset")
+        const charset = HttpSource.findParam(accept, ";", "charset")
         const decoder = new TextDecoder(charset)
         return decoder.decode(buffer) as T
       } else {
@@ -73,7 +66,7 @@ export abstract class HttpCaseSource<S> extends CaseSource<S> {
     }
   }
 
-  protected async submitForm<T>(url: string, obj: object, headers = {}): Promise<T> {
+  async submitForm<T>(url: string, obj: object, headers = {}): Promise<T> {
     const formData = new FormData()
     Object.entries(obj).forEach(entry => formData.append(entry[0], encodeURIComponent(entry[1])))
     const init: RequestInit = {method: "POST", headers, body: formData}

time/datasource/acufo/AcufoCase.ts

@@ -122,6 +122,8 @@ export type AcufoCase = {
    * "ACUFO-1945-07-16-NUMAZU-1"
    */
   caseNumber: string
+
+  url: URL
   summary: string
   data: AcufoCaseData
   sources: AcufoCaseSource[]

time/datasource/acufo/AcufoDatasource.ts

@@ -1,18 +1,25 @@
 import { RR0SsgContext } from "../../../RR0SsgContext"
-import { HttpCaseSource } from "../HttpCaseSource"
+import { HttpSource } from "../HttpSource"
 import { UrlUtil } from "../../../util/url/UrlUtil"
 import { JSDOM } from "jsdom"
 import { ObjectUtil } from "../../../util/ObjectUtil"
 import assert from "assert"
 import { AcufoCase } from "./AcufoCase"
+import { CaseSource } from "../CaseSource"
+import { NuforcState } from "../nuforc/NuforcState"
+import { NuforcCountry } from "../nuforc/NuforcCountry"
+
+import { NuforcShape } from "../nuforc/NuforcShape"
 
 interface QueryParameters {
 }
 
-export class AcufoDatasource extends HttpCaseSource<AcufoCase> {
+export class AcufoDatasource extends HttpSource implements CaseSource<AcufoCase> {
+  readonly author = "Gross, Patrick"
+  readonly copyright = "ACUFO/ALSACAT (Les ovnis vus de près)"
 
   constructor(readonly baseUrl = "https://ufologie.patrickgross.org", readonly searchPath = "alsacat") {
-    super("Gross, Patrick", "ACUFO/ALSACAT (Les ovnis vus de près)")
+    super()
   }
 
   async getAll(context: RR0SsgContext): Promise<AcufoCase[]> {
@@ -51,7 +58,7 @@ export class AcufoDatasource extends HttpCaseSource<AcufoCase> {
     dateTime.setHour(hour)
     dateTime.setMinutes(minutes)
     const url = new URL(caseLink.href, this.baseUrl)
-    const caseNumber = parseInt(HttpCaseSource.findParam(url.href, "?", "id"), 10)
+    const caseNumber = HttpSource.findParam(url.href, "?", "id")
     const stateStr = fields[3].textContent
     const state = ObjectUtil.enumFromValue<NuforcState>(NuforcState, stateStr)
     const countryStr = fields[4].textContent