commit 0d1c801720a54fd787d57dd66684779ab9e0a21c Author: Orion Kindel Date: Tue May 14 11:41:29 2024 -0500 feat: copy csv-stream diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..639514a --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +bower_components/ +node_modules/ +.pulp-cache/ +output/ +output-es/ +generated-docs/ +.psc-package/ +.psc* +.purs* +.psa* +.spago +.tmp/ diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..347e5e8 --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +purescript 0.15.15 diff --git a/README.md b/README.md new file mode 100644 index 0000000..ee047d8 --- /dev/null +++ b/README.md @@ -0,0 +1,81 @@ +# purescript-csv-stream + +Type-safe bindings for the streaming API of `csv-parse` and `csv-stringify`. + +## Installing +```bash +spago install csv-stream +{bun|yarn|npm|pnpm} install csv-parse csv-stringify +``` + +## Examples +### Stream +```purescript +module Main where + +import Prelude + +import Effect (Effect) +import Effect.Class (liftEffect) +import Effect.Aff (launchAff_) +import Node.Stream (pipe) +import Node.Stream as Stream +import Node.Stream.CSV.Stringify as CSV.Stringify +import Node.Stream.CSV.Parse as CSV.Parse + +type MyCSVType1 = {a :: Int, b :: Int, bar :: String, baz :: Boolean} +type MyCSVType2 = {ab :: Int, bar :: String, baz :: Boolean} + +atob :: MyCSVType1 -> MyCSVType2 +atob {a, b, bar, baz} = {ab: a + b, bar, baz} + +myCSV :: String +myCSV = "a,b,bar,baz\n1,2,\"hello, world!\",true\n3,3,,f" + +main :: Effect Unit +main = launchAff_ do + parser <- liftEffect $ CSV.Parse.make {} + stringifier <- liftEffect $ CSV.Stringify.make {} + + input <- liftEffect $ Stream.readableFromString myCSV + liftEffect $ Stream.pipe input parser + + records <- CSV.Parse.readAll parser + liftEffect $ for_ records \r -> CSV.Stringify.write $ atob r + liftEffect $ Stream.end stringifier + + -- "ab,bar,baz\n3,\"hello, world!\",true\n6,,false" + csvString <- CSV.Stringify.readAll stringifier + pure unit +``` + +### Synchronous +```purescript +module Main where + +import Prelude + +import Effect (Effect) +import Effect.Class (liftEffect) +import Effect.Aff (launchAff_) +import Node.Stream (pipe) +import Node.Stream as Stream +import Node.Stream.CSV.Stringify as CSV.Stringify +import Node.Stream.CSV.Parse as CSV.Parse + +type MyCSVType1 = {a :: Int, b :: Int, bar :: String, baz :: Boolean} +type MyCSVType2 = {ab :: Int, bar :: String, baz :: Boolean} + +atob :: MyCSVType1 -> MyCSVType2 +atob {a, b, bar, baz} = {ab: a + b, bar, baz} + +myCSV :: String +myCSV = "a,b,bar,baz\n1,2,\"hello, world!\",true\n3,3,,f" + +main :: Effect Unit +main = launchAff_ do + records :: Array MyCSVType1 <- CSV.Parse.parse myCSV + -- "ab,bar,baz\n3,\"hello, world!\",true\n6,,false" + csvString <- CSV.Stringify.stringify (atob <$> records) + pure unit +``` diff --git a/bun.lockb b/bun.lockb new file mode 100755 index 0000000..15b27c4 Binary files /dev/null and b/bun.lockb differ diff --git a/bun/fmt.js b/bun/fmt.js new file mode 100644 index 0000000..29d0fe0 --- /dev/null +++ b/bun/fmt.js @@ -0,0 +1,27 @@ +/** @type {(parser: string, ps: string[]) => import("bun").Subprocess} */ +const prettier = (parser, ps) => + Bun.spawn(["bun", "x", "prettier", "--write", "--parser", parser, ...ps], { + stdout: "inherit", + stderr: "inherit", + }); + +const procs = [ + prettier("babel", ["./src/**/*.js", "./bun/**/*.js", "./.prettierrc.cjs"]), + prettier("json", ["./package.json", "./jsconfig.json"]), + Bun.spawn( + [ + "bun", + "x", + "purs-tidy", + "format-in-place", + "src/**/*.purs", + "test/**/*.purs", + ], + { + stdout: "inherit", + stderr: "inherit", + }, + ), +]; + +await Promise.all(procs.map((p) => p.exited)); diff --git a/bun/prepare.js b/bun/prepare.js new file mode 100644 index 0000000..c64ce90 --- /dev/null +++ b/bun/prepare.js @@ -0,0 +1,34 @@ +import { readFile, writeFile } from "fs/promises"; +import { execSync } from "child_process"; + +let ver = process.argv[2]; +if (!ver) { + console.error(`tag required: bun bun/prepare.js v1.0.0`); + process.exit(1); +} else if (!/v\d+\.\d+\.\d+/.test(ver)) { + console.error(`invalid tag: ${ver}`); + process.exit(1); +} + +ver = (/\d+\.\d+\.\d+/.exec(ver) || [])[0] || ""; + +const pkg = await readFile("./package.json", "utf8"); +const pkgnew = pkg.replace(/"version": ".+"/, `"version": "v${ver}"`); +await writeFile("./package.json", pkgnew); + +const spago = await readFile("./spago.yaml", "utf8"); +const spagonew = spago.replace(/version: .+/, `version: '${ver}'`); +await writeFile("./spago.yaml", spagonew); + +const readme = await readFile("./README.md", "utf8"); +const readmenew = readme.replace( + /packages\/purescript-csv-stream\/.+?\//g, + `/packages/purescript-csv-stream/${ver}/`, +); +await writeFile("./README.md", readmenew); + +execSync(`git add spago.yaml package.json README.md`); +execSync(`git commit -m 'chore: prepare v${ver}'`); +execSync(`git tag v${ver}`); +execSync(`git push --tags`); +execSync(`git push --mirror github-mirror`); diff --git a/jsconfig.json b/jsconfig.json new file mode 100644 index 0000000..f48b93c --- /dev/null +++ b/jsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "types": ["bun-types"], + "lib": ["esnext"], + "target": "esnext", + "module": "esnext", + "moduleResolution": "bundler", + "moduleDetection": "force", + "jsx": "react", + "allowJs": true, + "checkJs": true, + "noEmit": true, + "strict": true + }, + "include": ["src/**/*.js", "bun/**/*.js"] +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..5d46b88 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "purescript-csv-stream", + "version": "v2.0.1", + "type": "module", + "dependencies": { + "csv-parse": "^5.5.5", + "csv-stringify": "^6.4.6", + "decimal.js": "^10.4.3" + }, + "devDependencies": { + "typescript": "^5.4.5" + } +} diff --git a/spago.lock b/spago.lock new file mode 100644 index 0000000..7cf26ff --- /dev/null +++ b/spago.lock @@ -0,0 +1,1076 @@ +workspace: + packages: + csv-stream: + path: ./ + dependencies: + - aff: ">=7.1.0 <8.0.0" + - arrays: ">=7.3.0 <8.0.0" + - bifunctors: ">=6.0.0 <7.0.0" + - datetime: ">=6.1.0 <7.0.0" + - effect: ">=4.0.0 <5.0.0" + - exceptions: ">=6.0.0 <7.0.0" + - foldable-traversable: ">=6.0.0 <7.0.0" + - foreign: ">=7.0.0 <8.0.0" + - foreign-object: ">=4.1.0 <5.0.0" + - integers: ">=6.0.0 <7.0.0" + - lists: ">=7.0.0 <8.0.0" + - maybe: ">=6.0.0 <7.0.0" + - newtype: ">=5.0.0 <6.0.0" + - node-buffer: ">=9.0.0 <10.0.0" + - node-event-emitter: ">=3.0.0 <4.0.0" + - node-stream-pipes: ">=1.3.0 <2.0.0" + - node-streams: ">=9.0.0 <10.0.0" + - nullable: ">=6.0.0 <7.0.0" + - numbers: ">=9.0.1 <10.0.0" + - ordered-collections: ">=3.2.0 <4.0.0" + - pipes: ">=8.0.0 <9.0.0" + - precise-datetime: ">=7.0.0 <8.0.0" + - prelude: ">=6.0.1 <7.0.0" + - record: ">=4.0.0 <5.0.0" + - record-extra: ">=5.0.1 <6.0.0" + - st: ">=6.2.0 <7.0.0" + - strings: ">=6.0.1 <7.0.0" + - tailrec: ">=6.1.0 <7.0.0" + - transformers: ">=6.0.0 <7.0.0" + - tuples: ">=7.0.0 <8.0.0" + - typelevel-prelude: ">=7.0.0 <8.0.0" + - unsafe-coerce: ">=6.0.0 <7.0.0" + test_dependencies: + - console + - gen + - node-fs + - node-zlib + - quickcheck + - simple-json + - spec + build_plan: + - aff + - ansi + - arraybuffer-types + - arrays + - avar + - bifunctors + - catenable-lists + - console + - const + - contravariant + - control + - datetime + - decimals + - distributive + - effect + - either + - enums + - exceptions + - exists + - fixed-points + - foldable-traversable + - foreign + - foreign-object + - fork + - formatters + - free + - functions + - functors + - gen + - identity + - integers + - invariant + - js-date + - lazy + - lcg + - lists + - maybe + - mmorph + - newtype + - node-buffer + - node-event-emitter + - node-fs + - node-path + - node-stream-pipes + - node-streams + - node-zlib + - nonempty + - now + - nullable + - numbers + - ordered-collections + - orders + - parallel + - parsing + - partial + - pipes + - precise-datetime + - prelude + - profunctor + - quickcheck + - random + - record + - record-extra + - refs + - safe-coerce + - simple-json + - spec + - st + - strings + - tailrec + - transformers + - tuples + - type-equality + - typelevel-prelude + - unfoldable + - unicode + - unordered-collections + - unsafe-coerce + - variant + extra_packages: {} +packages: + aff: + type: registry + version: 7.1.0 + integrity: sha256-7hOC6uQO9XBAI5FD8F33ChLjFAiZVfd4BJMqlMh7TNU= + dependencies: + - arrays + - bifunctors + - control + - datetime + - effect + - either + - exceptions + - foldable-traversable + - functions + - maybe + - newtype + - parallel + - prelude + - refs + - tailrec + - transformers + - unsafe-coerce + ansi: + type: registry + version: 7.0.0 + integrity: sha256-ZMB6HD+q9CXvn9fRCmJ8dvuDrOVHcjombL3oNOerVnE= + dependencies: + - foldable-traversable + - lists + - strings + arraybuffer-types: + type: registry + version: 3.0.2 + integrity: sha256-mQKokysYVkooS4uXbO+yovmV/s8b138Ws3zQvOwIHRA= + dependencies: [] + arrays: + type: registry + version: 7.3.0 + integrity: sha256-tmcklBlc/muUtUfr9RapdCPwnlQeB3aSrC4dK85gQlc= + dependencies: + - bifunctors + - control + - foldable-traversable + - functions + - maybe + - nonempty + - partial + - prelude + - safe-coerce + - st + - tailrec + - tuples + - unfoldable + - unsafe-coerce + avar: + type: registry + version: 5.0.0 + integrity: sha256-e7hf0x4hEpcygXP0LtvfvAQ49Bbj2aWtZT3gqM///0A= + dependencies: + - aff + - effect + - either + - exceptions + - functions + - maybe + bifunctors: + type: registry + version: 6.0.0 + integrity: sha256-/gZwC9YhNxZNQpnHa5BIYerCGM2jeX9ukZiEvYxm5Nw= + dependencies: + - const + - either + - newtype + - prelude + - tuples + catenable-lists: + type: registry + version: 7.0.0 + integrity: sha256-76vYENhwF4BWTBsjeLuErCH2jqVT4M3R1HX+4RwSftA= + dependencies: + - control + - foldable-traversable + - lists + - maybe + - prelude + - tuples + - unfoldable + console: + type: registry + version: 6.1.0 + integrity: sha256-CxmAzjgyuGDmt9FZW51VhV6rBPwR6o0YeKUzA9rSzcM= + dependencies: + - effect + - prelude + const: + type: registry + version: 6.0.0 + integrity: sha256-tNrxDW8D8H4jdHE2HiPzpLy08zkzJMmGHdRqt5BQuTc= + dependencies: + - invariant + - newtype + - prelude + contravariant: + type: registry + version: 6.0.0 + integrity: sha256-TP+ooAp3vvmdjfQsQJSichF5B4BPDHp3wAJoWchip6c= + dependencies: + - const + - either + - newtype + - prelude + - tuples + control: + type: registry + version: 6.0.0 + integrity: sha256-sH7Pg9E96JCPF9PIA6oQ8+BjTyO/BH1ZuE/bOcyj4Jk= + dependencies: + - newtype + - prelude + datetime: + type: registry + version: 6.1.0 + integrity: sha256-g/5X5BBegQWLpI9IWD+sY6mcaYpzzlW5lz5NBzaMtyI= + dependencies: + - bifunctors + - control + - either + - enums + - foldable-traversable + - functions + - gen + - integers + - lists + - maybe + - newtype + - numbers + - ordered-collections + - partial + - prelude + - tuples + decimals: + type: registry + version: 7.1.0 + integrity: sha256-DriR6lPEfFpjVv7e4JAQkr3ZLf0h17Qg2cAIrhxWV7w= + dependencies: + - maybe + distributive: + type: registry + version: 6.0.0 + integrity: sha256-HTDdmEnzigMl+02SJB88j+gAXDx9VKsbvR4MJGDPbOQ= + dependencies: + - identity + - newtype + - prelude + - tuples + - type-equality + effect: + type: registry + version: 4.0.0 + integrity: sha256-eBtZu+HZcMa5HilvI6kaDyVX3ji8p0W9MGKy2K4T6+M= + dependencies: + - prelude + either: + type: registry + version: 6.1.0 + integrity: sha256-6hgTPisnMWVwQivOu2PKYcH8uqjEOOqDyaDQVUchTpY= + dependencies: + - control + - invariant + - maybe + - prelude + enums: + type: registry + version: 6.0.1 + integrity: sha256-HWaD73JFLorc4A6trKIRUeDMdzE+GpkJaEOM1nTNkC8= + dependencies: + - control + - either + - gen + - maybe + - newtype + - nonempty + - partial + - prelude + - tuples + - unfoldable + exceptions: + type: registry + version: 6.0.0 + integrity: sha256-y/xTAEIZIARCE+50/u1di0ncebJ+CIwNOLswyOWzMTw= + dependencies: + - effect + - either + - maybe + - prelude + exists: + type: registry + version: 6.0.0 + integrity: sha256-A0JQHpTfo1dNOj9U5/Fd3xndlRSE0g2IQWOGor2yXn8= + dependencies: + - unsafe-coerce + fixed-points: + type: registry + version: 7.0.0 + integrity: sha256-hTl5fzeG4mzAOFzEzAeNH7kJvJgYCH7x3v2NdX9pOE4= + dependencies: + - exists + - newtype + - prelude + - transformers + foldable-traversable: + type: registry + version: 6.0.0 + integrity: sha256-fLeqRYM4jUrZD5H4WqcwUgzU7XfYkzO4zhgtNc3jcWM= + dependencies: + - bifunctors + - const + - control + - either + - functors + - identity + - maybe + - newtype + - orders + - prelude + - tuples + foreign: + type: registry + version: 7.0.0 + integrity: sha256-1ORiqoS3HW+qfwSZAppHPWy4/6AQysxZ2t29jcdUMNA= + dependencies: + - either + - functions + - identity + - integers + - lists + - maybe + - prelude + - strings + - transformers + foreign-object: + type: registry + version: 4.1.0 + integrity: sha256-q24okj6mT+yGHYQ+ei/pYPj5ih6sTbu7eDv/WU56JVo= + dependencies: + - arrays + - foldable-traversable + - functions + - gen + - lists + - maybe + - prelude + - st + - tailrec + - tuples + - typelevel-prelude + - unfoldable + fork: + type: registry + version: 6.0.0 + integrity: sha256-X7u0SuCvFbLbzuNEKLBNuWjmcroqMqit4xEzpQwAP7E= + dependencies: + - aff + formatters: + type: registry + version: 7.0.0 + integrity: sha256-5JaC9d2p0xoqJWjWxlHH19R4iJwFTBr4j7SlYcLgicE= + dependencies: + - datetime + - fixed-points + - lists + - numbers + - parsing + - prelude + - transformers + free: + type: registry + version: 7.1.0 + integrity: sha256-JAumgEsGSzJCNLD8AaFvuX7CpqS5yruCngi6yI7+V5k= + dependencies: + - catenable-lists + - control + - distributive + - either + - exists + - foldable-traversable + - invariant + - lazy + - maybe + - prelude + - tailrec + - transformers + - tuples + - unsafe-coerce + functions: + type: registry + version: 6.0.0 + integrity: sha256-adMyJNEnhGde2unHHAP79gPtlNjNqzgLB8arEOn9hLI= + dependencies: + - prelude + functors: + type: registry + version: 5.0.0 + integrity: sha256-zfPWWYisbD84MqwpJSZFlvM6v86McM68ob8p9s27ywU= + dependencies: + - bifunctors + - const + - contravariant + - control + - distributive + - either + - invariant + - maybe + - newtype + - prelude + - profunctor + - tuples + - unsafe-coerce + gen: + type: registry + version: 4.0.0 + integrity: sha256-f7yzAXWwr+xnaqEOcvyO3ezKdoes8+WXWdXIHDBCAPI= + dependencies: + - either + - foldable-traversable + - identity + - maybe + - newtype + - nonempty + - prelude + - tailrec + - tuples + - unfoldable + identity: + type: registry + version: 6.0.0 + integrity: sha256-4wY0XZbAksjY6UAg99WkuKyJlQlWAfTi2ssadH0wVMY= + dependencies: + - control + - invariant + - newtype + - prelude + integers: + type: registry + version: 6.0.0 + integrity: sha256-sf+sK26R1hzwl3NhXR7WAu9zCDjQnfoXwcyGoseX158= + dependencies: + - maybe + - numbers + - prelude + invariant: + type: registry + version: 6.0.0 + integrity: sha256-RGWWyYrz0Hs1KjPDA+87Kia67ZFBhfJ5lMGOMCEFoLo= + dependencies: + - control + - prelude + js-date: + type: registry + version: 8.0.0 + integrity: sha256-6TVF4DWg5JL+jRAsoMssYw8rgOVALMUHT1CuNZt8NRo= + dependencies: + - datetime + - effect + - exceptions + - foreign + - integers + - now + lazy: + type: registry + version: 6.0.0 + integrity: sha256-lMsfFOnlqfe4KzRRiW8ot5ge6HtcU3Eyh2XkXcP5IgU= + dependencies: + - control + - foldable-traversable + - invariant + - prelude + lcg: + type: registry + version: 4.0.0 + integrity: sha256-h7ME5cthLfbgJOJdsZcSfFpwXsx4rf8YmhebU+3iSYg= + dependencies: + - effect + - integers + - maybe + - partial + - prelude + - random + lists: + type: registry + version: 7.0.0 + integrity: sha256-EKF15qYqucuXP2lT/xPxhqy58f0FFT6KHdIB/yBOayI= + dependencies: + - bifunctors + - control + - foldable-traversable + - lazy + - maybe + - newtype + - nonempty + - partial + - prelude + - tailrec + - tuples + - unfoldable + maybe: + type: registry + version: 6.0.0 + integrity: sha256-5cCIb0wPwbat2PRkQhUeZO0jcAmf8jCt2qE0wbC3v2Q= + dependencies: + - control + - invariant + - newtype + - prelude + mmorph: + type: registry + version: 7.0.0 + integrity: sha256-urZlZNNqGeQFe5D/ClHlR8QgGBNHTMFPtJ5S5IpflTQ= + dependencies: + - free + - functors + - transformers + newtype: + type: registry + version: 5.0.0 + integrity: sha256-gdrQu8oGe9eZE6L3wOI8ql/igOg+zEGB5ITh2g+uttw= + dependencies: + - prelude + - safe-coerce + node-buffer: + type: registry + version: 9.0.0 + integrity: sha256-PWE2DJ5ruBLCmeA/fUiuySEFmUJ/VuRfyrnCuVZBlu4= + dependencies: + - arraybuffer-types + - effect + - maybe + - nullable + - st + - unsafe-coerce + node-event-emitter: + type: registry + version: 3.0.0 + integrity: sha256-Qw0MjsT4xRH2j2i4K8JmRjcMKnH5z1Cw39t00q4LE4w= + dependencies: + - effect + - either + - functions + - maybe + - nullable + - prelude + - unsafe-coerce + node-fs: + type: registry + version: 9.1.0 + integrity: sha256-TzhvGdrwcM0bazDvrWSqh+M/H8GKYf1Na6aGm2Qg4+c= + dependencies: + - datetime + - effect + - either + - enums + - exceptions + - functions + - integers + - js-date + - maybe + - node-buffer + - node-path + - node-streams + - nullable + - partial + - prelude + - strings + - unsafe-coerce + node-path: + type: registry + version: 5.0.0 + integrity: sha256-pd82nQ+2l5UThzaxPdKttgDt7xlsgIDLpPG0yxDEdyE= + dependencies: + - effect + node-stream-pipes: + type: registry + version: 1.3.0 + integrity: sha256-5Jpf0BLn0ExQWYxbTTewai4M8quEmEVHxihc9CM1Juo= + dependencies: + - aff + - arrays + - effect + - either + - exceptions + - foldable-traversable + - foreign-object + - lists + - maybe + - mmorph + - newtype + - node-buffer + - node-event-emitter + - node-fs + - node-path + - node-streams + - node-zlib + - ordered-collections + - parallel + - pipes + - prelude + - st + - strings + - tailrec + - transformers + - tuples + - unordered-collections + - unsafe-coerce + node-streams: + type: registry + version: 9.0.0 + integrity: sha256-2n6dq7YWleTDmD1Kur/ul7Cn08IvWrScgPf+0PgX2TQ= + dependencies: + - aff + - effect + - either + - exceptions + - node-buffer + - node-event-emitter + - nullable + - prelude + node-zlib: + type: registry + version: 0.4.0 + integrity: sha256-kYSajFQFzWVg71l5/y4w4kXdTr5EJoqyV3D2RqmAjQ4= + dependencies: + - aff + - effect + - either + - functions + - node-buffer + - node-streams + - prelude + - unsafe-coerce + nonempty: + type: registry + version: 7.0.0 + integrity: sha256-54ablJZUHGvvlTJzi3oXyPCuvY6zsrWJuH/dMJ/MFLs= + dependencies: + - control + - foldable-traversable + - maybe + - prelude + - tuples + - unfoldable + now: + type: registry + version: 6.0.0 + integrity: sha256-xZ7x37ZMREfs6GCDw/h+FaKHV/3sPWmtqBZRGTxybQY= + dependencies: + - datetime + - effect + nullable: + type: registry + version: 6.0.0 + integrity: sha256-yiGBVl3AD+Guy4kNWWeN+zl1gCiJK+oeIFtZtPCw4+o= + dependencies: + - effect + - functions + - maybe + numbers: + type: registry + version: 9.0.1 + integrity: sha256-/9M6aeMDBdB4cwYDeJvLFprAHZ49EbtKQLIJsneXLIk= + dependencies: + - functions + - maybe + ordered-collections: + type: registry + version: 3.2.0 + integrity: sha256-o9jqsj5rpJmMdoe/zyufWHFjYYFTTsJpgcuCnqCO6PM= + dependencies: + - arrays + - foldable-traversable + - gen + - lists + - maybe + - partial + - prelude + - st + - tailrec + - tuples + - unfoldable + orders: + type: registry + version: 6.0.0 + integrity: sha256-nBA0g3/ai0euH8q9pSbGqk53W2q6agm/dECZTHcoink= + dependencies: + - newtype + - prelude + parallel: + type: registry + version: 6.0.0 + integrity: sha256-VJbkGD0rAKX+NUEeBJbYJ78bEKaZbgow+QwQEfPB6ko= + dependencies: + - control + - effect + - either + - foldable-traversable + - functors + - maybe + - newtype + - prelude + - profunctor + - refs + - transformers + parsing: + type: registry + version: 10.2.0 + integrity: sha256-ZDIdMFAKkst57x6BVa1aUWJnS8smoZnXsZ339Aq1mPA= + dependencies: + - arrays + - control + - effect + - either + - enums + - foldable-traversable + - functions + - identity + - integers + - lazy + - lists + - maybe + - newtype + - nullable + - numbers + - partial + - prelude + - st + - strings + - tailrec + - transformers + - tuples + - unfoldable + - unicode + - unsafe-coerce + partial: + type: registry + version: 4.0.0 + integrity: sha256-fwXerld6Xw1VkReh8yeQsdtLVrjfGiVuC5bA1Wyo/J4= + dependencies: [] + pipes: + type: registry + version: 8.0.0 + integrity: sha256-kvfqGM4cPA/wCcBHbp5psouFw5dZGvku2462x7ZBwSY= + dependencies: + - aff + - lists + - mmorph + - prelude + - tailrec + - transformers + - tuples + precise-datetime: + type: registry + version: 7.0.0 + integrity: sha256-F7tzZ7++Ihtg3xjumzwaHQvGQg61UtEAe5MWeOlTzRY= + dependencies: + - arrays + - datetime + - decimals + - either + - enums + - foldable-traversable + - formatters + - integers + - js-date + - lists + - maybe + - newtype + - numbers + - prelude + - strings + - tuples + - unicode + prelude: + type: registry + version: 6.0.1 + integrity: sha256-o8p6SLYmVPqzXZhQFd2hGAWEwBoXl1swxLG/scpJ0V0= + dependencies: [] + profunctor: + type: registry + version: 6.0.1 + integrity: sha256-E58hSYdJvF2Qjf9dnWLPlJKh2Z2fLfFLkQoYi16vsFk= + dependencies: + - control + - distributive + - either + - exists + - invariant + - newtype + - prelude + - tuples + quickcheck: + type: registry + version: 8.0.1 + integrity: sha256-ZvpccKQCvgslTXZCNmpYW4bUsFzhZd/kQUr2WmxFTGY= + dependencies: + - arrays + - console + - control + - effect + - either + - enums + - exceptions + - foldable-traversable + - gen + - identity + - integers + - lazy + - lcg + - lists + - maybe + - newtype + - nonempty + - numbers + - partial + - prelude + - record + - st + - strings + - tailrec + - transformers + - tuples + - unfoldable + random: + type: registry + version: 6.0.0 + integrity: sha256-CJ611a35MPCE7XQMp0rdC6MCn76znlhisiCRgboAG+Q= + dependencies: + - effect + - integers + record: + type: registry + version: 4.0.0 + integrity: sha256-Za5U85bTRJEfGK5Sk4hM41oXy84YQI0I8TL3WUn1Qzg= + dependencies: + - functions + - prelude + - unsafe-coerce + record-extra: + type: registry + version: 5.0.1 + integrity: sha256-7vnREK2fpGJ7exswSeA9UpZFuU+UXRt3SA7AFUldT/Y= + dependencies: + - arrays + - functions + - lists + - prelude + - record + - tuples + - typelevel-prelude + refs: + type: registry + version: 6.0.0 + integrity: sha256-Vgwne7jIbD3ZMoLNNETLT8Litw6lIYo3MfYNdtYWj9s= + dependencies: + - effect + - prelude + safe-coerce: + type: registry + version: 2.0.0 + integrity: sha256-a1ibQkiUcbODbLE/WAq7Ttbbh9ex+x33VCQ7GngKudU= + dependencies: + - unsafe-coerce + simple-json: + type: registry + version: 9.0.0 + integrity: sha256-K3RJaThqsszTd+TEklzZmAdDqvIHWgXIfKqlsoykU1c= + dependencies: + - arrays + - exceptions + - foreign + - foreign-object + - nullable + - prelude + - record + - typelevel-prelude + - variant + spec: + type: registry + version: 7.6.0 + integrity: sha256-+merGdQbL9zWONbnt8S8J9afGJ59MQqGtS0qSd3yu4I= + dependencies: + - aff + - ansi + - arrays + - avar + - bifunctors + - control + - datetime + - effect + - either + - exceptions + - foldable-traversable + - fork + - identity + - integers + - lists + - maybe + - newtype + - now + - ordered-collections + - parallel + - pipes + - prelude + - refs + - strings + - tailrec + - transformers + - tuples + st: + type: registry + version: 6.2.0 + integrity: sha256-z9X0WsOUlPwNx9GlCC+YccCyz8MejC8Wb0C4+9fiBRY= + dependencies: + - partial + - prelude + - tailrec + - unsafe-coerce + strings: + type: registry + version: 6.0.1 + integrity: sha256-WssD3DbX4OPzxSdjvRMX0yvc9+pS7n5gyPv5I2Trb7k= + dependencies: + - arrays + - control + - either + - enums + - foldable-traversable + - gen + - integers + - maybe + - newtype + - nonempty + - partial + - prelude + - tailrec + - tuples + - unfoldable + - unsafe-coerce + tailrec: + type: registry + version: 6.1.0 + integrity: sha256-Xx19ECVDRrDWpz9D2GxQHHV89vd61dnXxQm0IcYQHGk= + dependencies: + - bifunctors + - effect + - either + - identity + - maybe + - partial + - prelude + - refs + transformers: + type: registry + version: 6.0.0 + integrity: sha256-Pzw40HjthX77tdPAYzjx43LK3X5Bb7ZspYAp27wksFA= + dependencies: + - control + - distributive + - effect + - either + - exceptions + - foldable-traversable + - identity + - lazy + - maybe + - newtype + - prelude + - tailrec + - tuples + - unfoldable + tuples: + type: registry + version: 7.0.0 + integrity: sha256-1rXgTomes9105BjgXqIw0FL6Fz1lqqUTLWOumhWec1M= + dependencies: + - control + - invariant + - prelude + type-equality: + type: registry + version: 4.0.1 + integrity: sha256-Hs9D6Y71zFi/b+qu5NSbuadUQXe5iv5iWx0226vOHUw= + dependencies: [] + typelevel-prelude: + type: registry + version: 7.0.0 + integrity: sha256-uFF2ph+vHcQpfPuPf2a3ukJDFmLhApmkpTMviHIWgJM= + dependencies: + - prelude + - type-equality + unfoldable: + type: registry + version: 6.0.0 + integrity: sha256-JtikvJdktRap7vr/K4ITlxUX1QexpnqBq0G/InLr6eg= + dependencies: + - foldable-traversable + - maybe + - partial + - prelude + - tuples + unicode: + type: registry + version: 6.0.0 + integrity: sha256-QJnTVZpmihEAUiMeYrfkusVoziJWp2hJsgi9bMQLue8= + dependencies: + - foldable-traversable + - maybe + - strings + unordered-collections: + type: registry + version: 3.1.0 + integrity: sha256-H2eQR+ylI+cljz4XzWfEbdF7ee+pnw2IZCeq69AuJ+Q= + dependencies: + - arrays + - enums + - functions + - integers + - lists + - prelude + - record + - tuples + - typelevel-prelude + - unfoldable + unsafe-coerce: + type: registry + version: 6.0.0 + integrity: sha256-IqIYW4Vkevn8sI+6aUwRGvd87tVL36BBeOr0cGAE7t0= + dependencies: [] + variant: + type: registry + version: 8.0.0 + integrity: sha256-SR//zQDg2dnbB8ZHslcxieUkCeNlbMToapvmh9onTtw= + dependencies: + - enums + - lists + - maybe + - partial + - prelude + - record + - tuples + - unsafe-coerce diff --git a/spago.yaml b/spago.yaml new file mode 100644 index 0000000..299c8ae --- /dev/null +++ b/spago.yaml @@ -0,0 +1,56 @@ +package: + name: csv-stream + publish: + version: '2.0.1' + license: 'GPL-3.0-or-later' + location: + githubOwner: 'cakekindel' + githubRepo: 'purescript-csv-stream' + build: + strict: true + pedanticPackages: true + dependencies: + - node-stream-pipes: ">=1.3.0 <2.0.0" + - aff: ">=7.1.0 <8.0.0" + - arrays: ">=7.3.0 <8.0.0" + - bifunctors: ">=6.0.0 <7.0.0" + - datetime: ">=6.1.0 <7.0.0" + - effect: ">=4.0.0 <5.0.0" + - exceptions: ">=6.0.0 <7.0.0" + - foldable-traversable: ">=6.0.0 <7.0.0" + - foreign: ">=7.0.0 <8.0.0" + - foreign-object: ">=4.1.0 <5.0.0" + - integers: ">=6.0.0 <7.0.0" + - lists: ">=7.0.0 <8.0.0" + - maybe: ">=6.0.0 <7.0.0" + - newtype: ">=5.0.0 <6.0.0" + - node-buffer: ">=9.0.0 <10.0.0" + - node-event-emitter: ">=3.0.0 <4.0.0" + - node-streams: ">=9.0.0 <10.0.0" + - nullable: ">=6.0.0 <7.0.0" + - numbers: ">=9.0.1 <10.0.0" + - ordered-collections: ">=3.2.0 <4.0.0" + - pipes: ">=8.0.0 <9.0.0" + - precise-datetime: ">=7.0.0 <8.0.0" + - prelude: ">=6.0.1 <7.0.0" + - record: ">=4.0.0 <5.0.0" + - record-extra: ">=5.0.1 <6.0.0" + - st: ">=6.2.0 <7.0.0" + - strings: ">=6.0.1 <7.0.0" + - tailrec: ">=6.1.0 <7.0.0" + - transformers: ">=6.0.0 <7.0.0" + - tuples: ">=7.0.0 <8.0.0" + - typelevel-prelude: ">=7.0.0 <8.0.0" + - unsafe-coerce: ">=6.0.0 <7.0.0" + test: + main: Test.Main + dependencies: + - console + - gen + - node-fs + - node-zlib + - quickcheck + - simple-json + - spec +workspace: + extraPackages: {} diff --git a/src/Data.CSV.Record.purs b/src/Data.CSV.Record.purs new file mode 100644 index 0000000..8728d3d --- /dev/null +++ b/src/Data.CSV.Record.purs @@ -0,0 +1,50 @@ +module Data.CSV.Record where + +import Prelude + +import Control.Monad.Error.Class (liftMaybe) +import Control.Monad.Except (Except) +import Data.Array as Array +import Data.CSV (class ReadCSV, class WriteCSV, readCSV, writeCSV) +import Data.List.NonEmpty (NonEmptyList) +import Data.Map (Map) +import Data.Map as Map +import Data.Maybe (fromMaybe) +import Data.Symbol (class IsSymbol, reflectSymbol) +import Foreign (ForeignError(..)) +import Prim.Row (class Cons, class Lacks) +import Prim.RowList (class RowToList, Cons, Nil, RowList) +import Record as Record +import Type.Prelude (Proxy(..)) + +class WriteCSVRecord :: Row Type -> RowList Type -> Constraint +class RowToList r rl <= WriteCSVRecord r rl | rl -> r where + writeCSVRecord :: { | r } -> Array String + +instance (RowToList r (Cons k v tailrl), IsSymbol k, WriteCSV v, Lacks k tail, Cons k v tail r, WriteCSVRecord tail tailrl) => WriteCSVRecord r (Cons k v tailrl) where + writeCSVRecord r = + let + val = writeCSV $ Record.get (Proxy @k) r + tail = writeCSVRecord @tail @tailrl $ Record.delete (Proxy @k) r + in + [ val ] <> tail + +instance WriteCSVRecord () Nil where + writeCSVRecord _ = [] + +class ReadCSVRecord :: Row Type -> RowList Type -> Constraint +class RowToList r rl <= ReadCSVRecord r rl | rl -> r where + readCSVRecord :: Map String Int -> Array String -> Except (NonEmptyList ForeignError) { | r } + +instance (RowToList r (Cons k v tailrl), IsSymbol k, ReadCSV v, Lacks k tail, Cons k v tail r, ReadCSVRecord tail tailrl) => ReadCSVRecord r (Cons k v tailrl) where + readCSVRecord cols vals = do + let + k = reflectSymbol (Proxy @k) + pos <- liftMaybe (pure $ ForeignError $ "reached end of row looking for column " <> k) $ Map.lookup k cols + let valraw = fromMaybe "" $ Array.index vals pos + val <- readCSV @v valraw + tail <- readCSVRecord @tail @tailrl cols vals + pure $ Record.insert (Proxy @k) val tail + +instance ReadCSVRecord () Nil where + readCSVRecord _ _ = pure {} diff --git a/src/Data.CSV.purs b/src/Data.CSV.purs new file mode 100644 index 0000000..56eb679 --- /dev/null +++ b/src/Data.CSV.purs @@ -0,0 +1,74 @@ +module Data.CSV where + +import Prelude + +import Control.Monad.Error.Class (liftMaybe, throwError) +import Control.Monad.Except (Except) +import Data.DateTime (DateTime) +import Data.Int as Int +import Data.List.NonEmpty (NonEmptyList) +import Data.Maybe (Maybe(..), maybe) +import Data.Newtype (unwrap) +import Data.Number (fromString) as Number +import Data.Number.Format (toString) as Number +import Data.PreciseDateTime (fromDateTime, fromRFC3339String, toDateTimeLossy, toRFC3339String) +import Data.RFC3339String (RFC3339String(..)) +import Data.String as String +import Foreign (ForeignError(..)) + +class ReadCSV a where + readCSV :: String -> Except (NonEmptyList ForeignError) a + +class WriteCSV a where + writeCSV :: a -> String + +instance ReadCSV Int where + readCSV s = liftMaybe (pure $ ForeignError $ "invalid integer: " <> s) $ Int.fromString s + +instance ReadCSV Number where + readCSV s = liftMaybe (pure $ ForeignError $ "invalid number: " <> s) $ Number.fromString s + +instance ReadCSV String where + readCSV = pure + +instance ReadCSV DateTime where + readCSV s = map toDateTimeLossy $ liftMaybe (pure $ ForeignError $ "invalid ISO date string: " <> s) $ fromRFC3339String $ RFC3339String s + +instance ReadCSV Boolean where + readCSV s = + let + inner "t" = pure true + inner "true" = pure true + inner "yes" = pure true + inner "y" = pure true + inner "1" = pure true + inner "f" = pure false + inner "false" = pure false + inner "no" = pure false + inner "n" = pure false + inner "0" = pure false + inner _ = throwError $ pure $ ForeignError $ "invalid boolean value: " <> s + in + inner $ String.toLower s + +instance ReadCSV a => ReadCSV (Maybe a) where + readCSV "" = pure Nothing + readCSV s = Just <$> readCSV s + +instance WriteCSV Int where + writeCSV = Int.toStringAs Int.decimal + +instance WriteCSV Number where + writeCSV = Number.toString + +instance WriteCSV String where + writeCSV = identity + +instance WriteCSV DateTime where + writeCSV = unwrap <<< toRFC3339String <<< fromDateTime + +instance WriteCSV Boolean where + writeCSV = show + +instance WriteCSV a => WriteCSV (Maybe a) where + writeCSV = maybe "" writeCSV diff --git a/src/Node.Stream.CSV.Parse.js b/src/Node.Stream.CSV.Parse.js new file mode 100644 index 0000000..a72708c --- /dev/null +++ b/src/Node.Stream.CSV.Parse.js @@ -0,0 +1,7 @@ +import { Parser } from "csv-parse"; + +/** @type {(s: import('csv-parse').Options) => () => Parser} */ +export const makeImpl = (c) => () => new Parser(c); + +/** @type {(s: Parser) => () => Array | null} */ +export const readImpl = (p) => () => p.read(); diff --git a/src/Node.Stream.CSV.Parse.purs b/src/Node.Stream.CSV.Parse.purs new file mode 100644 index 0000000..943f225 --- /dev/null +++ b/src/Node.Stream.CSV.Parse.purs @@ -0,0 +1,77 @@ +module Node.Stream.CSV.Parse where + +import Prelude hiding (join) + +import Data.Nullable (Nullable) +import Effect (Effect) +import Effect.Uncurried (mkEffectFn1) +import Foreign (Foreign, unsafeToForeign) +import Foreign.Object (Object) +import Foreign.Object (union) as Object +import Node.Buffer (Buffer) +import Node.EventEmitter (EventHandle(..)) +import Node.EventEmitter.UtilTypes (EventHandle1) +import Node.Stream (Read, Stream, Write) +import Node.Stream.Object (Transform) as Object +import Prim.Row (class Union) +import Unsafe.Coerce (unsafeCoerce) + +data CSVRead + +-- | Stream transforming chunks of a CSV file +-- | into parsed purescript objects. +-- | +-- | The CSV contents may be piped into this stream +-- | as Buffer or String chunks. +type CSVParser :: Row Type -> Type +type CSVParser r = Stream (read :: Read, write :: Write, csv :: CSVRead | r) + +-- | https://csv.js.org/parse/options/ +type Config r = + ( bom :: Boolean + , group_columns_by_name :: Boolean + , comment :: String + , comment_no_infix :: Boolean + , delimiter :: String + , encoding :: String + , escape :: String + , from :: Int + , from_line :: Int + , ignore_last_delimiters :: Boolean + , info :: Boolean + , max_record_size :: Int + , quote :: String + , raw :: Boolean + , record_delimiter :: String + , relax_column_count :: Boolean + , skip_empty_lines :: Boolean + , skip_records_with_empty_values :: Boolean + , skip_records_with_error :: Boolean + , to :: Int + , to_line :: Int + , trim :: Boolean + , ltrim :: Boolean + , rtrim :: Boolean + | r + ) + +-- | Create a CSVParser +make :: forall @config @missing @extra. Union config missing (Config extra) => { | config } -> Effect (CSVParser ()) +make = makeImpl <<< unsafeToForeign <<< Object.union (recordToForeign { columns: false, cast: false, cast_date: false }) <<< recordToForeign + +toObjectStream :: CSVParser () -> Object.Transform Buffer (Array String) +toObjectStream = unsafeCoerce + +-- | `data` event. Emitted when a CSV record has been parsed. +dataH :: forall a. EventHandle1 (CSVParser a) (Array String) +dataH = EventHandle "data" mkEffectFn1 + +-- | FFI +foreign import makeImpl :: forall r. Foreign -> Effect (Stream r) + +-- | FFI +foreign import readImpl :: forall r. Stream r -> Effect (Nullable (Array String)) + +-- | FFI +recordToForeign :: forall r. Record r -> Object Foreign +recordToForeign = unsafeCoerce diff --git a/src/Node.Stream.CSV.Stringify.js b/src/Node.Stream.CSV.Stringify.js new file mode 100644 index 0000000..99e4ed7 --- /dev/null +++ b/src/Node.Stream.CSV.Stringify.js @@ -0,0 +1,7 @@ +import { stringify } from "csv-stringify"; + +/** @type {(c: import('csv-stringify').Options) => () => import('csv-stringify').Stringifier} */ +export const makeImpl = (c) => () => stringify(c); + +/** @type {(s: import('csv-stringify').Stringifier) => (vals: Array) => () => void} */ +export const writeImpl = (s) => (vals) => () => s.write(vals); diff --git a/src/Node.Stream.CSV.Stringify.purs b/src/Node.Stream.CSV.Stringify.purs new file mode 100644 index 0000000..2a3a605 --- /dev/null +++ b/src/Node.Stream.CSV.Stringify.purs @@ -0,0 +1,83 @@ +module Node.Stream.CSV.Stringify where + +import Prelude + +import Data.CSV.Record (class WriteCSVRecord, writeCSVRecord) +import Data.String.Regex (Regex) +import Effect (Effect) +import Foreign (Foreign, unsafeToForeign) +import Foreign.Object (Object) +import Foreign.Object (union) as Object +import Node.Stream (Read, Stream, Write) +import Node.Stream.Object (Transform) as Object +import Prim.Row (class Union) +import Prim.RowList (class RowToList) +import Unsafe.Coerce (unsafeCoerce) + +data CSVWrite + +-- | Stream transforming rows of stringified CSV values +-- | to CSV-formatted rows. +-- | +-- | Write rows to the stream using `write`. +-- | +-- | Stringified rows are emitted on the `Readable` end as string +-- | chunks, meaning it can be treated as a `Node.Stream.Readable` +-- | that has had `setEncoding UTF8` invoked on it. +type CSVStringifier :: Row Type -> Type +type CSVStringifier r = Stream (read :: Read, write :: Write, csv :: CSVWrite | r) + +-- | https://csv.js.org/stringify/options/ +type Config r = + ( bom :: Boolean + , delimiter :: String + , record_delimiter :: String + , escape :: String + , escape_formulas :: Boolean + , quote :: String + , quoted :: Boolean + , quoted_empty :: Boolean + , quoted_match :: Regex + , quoted_string :: Boolean + | r + ) + +foreign import makeImpl :: forall r. Foreign -> Effect (Stream r) +foreign import writeImpl :: forall r. Stream r -> Array String -> Effect Unit + +recordToForeign :: forall r. Record r -> Object Foreign +recordToForeign = unsafeCoerce + +-- | Create a raw Transform stream that accepts chunks of `Array String`, +-- | and transforms them into string CSV rows. +-- | +-- | Requires an ordered array of column names. +make + :: forall @config @missing @extra + . Union config missing (Config extra) + => Array String + -> { | config } + -> Effect (CSVStringifier ()) +make columns = + makeImpl + <<< unsafeToForeign + <<< Object.union (recordToForeign { columns, header: true }) + <<< recordToForeign + +-- | Convert the raw stream to a typed ObjectStream +toObjectStream :: CSVStringifier () -> Object.Transform (Array String) String +toObjectStream = unsafeCoerce + +-- | Write a record to a CSVStringifier. +-- | +-- | The record will be emitted on the `Readable` end +-- | of the stream as a string chunk. +write :: forall @r rl a. RowToList r rl => WriteCSVRecord r rl => CSVStringifier a -> { | r } -> Effect Unit +write s = writeImpl s <<< writeCSVRecord @r @rl + +-- | Write a record to a CSVStringifier. +-- | +-- | The record will be emitted on the `Readable` end +-- | of the stream as a string chunk. +writeRaw :: forall a. CSVStringifier a -> Array String -> Effect Unit +writeRaw = writeImpl diff --git a/src/Node.Stream.CSV.purs b/src/Node.Stream.CSV.purs new file mode 100644 index 0000000..d47d642 --- /dev/null +++ b/src/Node.Stream.CSV.purs @@ -0,0 +1 @@ +module Node.Stream.CSV where diff --git a/src/Pipes.CSV.purs b/src/Pipes.CSV.purs new file mode 100644 index 0000000..9ac747c --- /dev/null +++ b/src/Pipes.CSV.purs @@ -0,0 +1,98 @@ +module Pipes.CSV where + +import Prelude + +import Control.Monad.Error.Class (liftEither) +import Control.Monad.Except (runExcept) +import Control.Monad.Rec.Class (forever) +import Control.Monad.ST.Global as ST +import Control.Monad.ST.Ref as STRef +import Data.Array as Array +import Data.Bifunctor (lmap) +import Data.CSV.Record (class ReadCSVRecord, class WriteCSVRecord, readCSVRecord, writeCSVRecord) +import Data.FunctorWithIndex (mapWithIndex) +import Data.Map as Map +import Data.Maybe (Maybe(..)) +import Data.Tuple.Nested ((/\)) +import Effect.Aff (Aff) +import Effect.Class (liftEffect) +import Effect.Exception (error) +import Node.Buffer (Buffer) +import Node.Stream.CSV.Parse as CSV.Parse +import Node.Stream.CSV.Stringify as CSV.Stringify +import Pipes (await, yield, (>->)) +import Pipes.Core (Pipe) +import Pipes.Node.Stream as Pipes.Stream +import Prim.RowList (class RowToList) +import Record.Extra (class Keys, keys) +import Type.Prelude (Proxy(..)) + +-- | Transforms buffer chunks of a CSV file to parsed +-- | records of `r`. +-- | +-- | ``` +-- | -- == my-data.csv.gz == +-- | -- id,foo,is_deleted +-- | -- 1,hi,f +-- | -- 2,bye,t +-- | +-- | rows +-- | :: Array {id :: Int, foo :: String, is_deleted :: Boolean} +-- | <- map Array.fromFoldable +-- | $ Pipes.toListM +-- | $ Pipes.Node.Stream.unEOS +-- | $ Pipes.Node.FS.read "my-data.csv.gz" +-- | >-> Pipes.Node.Zlib.gunzip +-- | >-> Pipes.CSV.parse +-- | rows `shouldEqual` [{id: 1, foo: "hi", is_deleted: false}, {id: 2, foo: "bye", is_deleted: true}] +-- | ``` +parse + :: forall @r rl + . RowToList r rl + => ReadCSVRecord r rl + => Pipe (Maybe Buffer) (Maybe { | r }) Aff Unit +parse = do + raw <- liftEffect $ CSV.Parse.make {} + colsST <- liftEffect $ ST.toEffect $ STRef.new Nothing + + let + readCols = liftEffect $ ST.toEffect $ STRef.read colsST + putCols a = void $ liftEffect $ ST.toEffect $ STRef.write (Just a) colsST + + parse' a cols' = liftEither $ lmap (error <<< show) $ runExcept $ readCSVRecord @r @rl cols' a + firstRow a = putCols $ Map.fromFoldable $ mapWithIndex (flip (/\)) a + row a cols' = yield =<< parse' a cols' + unmarshal = forever do + r <- await + cols <- readCols + case cols of + Just cols' -> row r cols' + Nothing -> firstRow r + + parser = Pipes.Stream.fromTransform $ CSV.Parse.toObjectStream raw + parser >-> Pipes.Stream.inEOS unmarshal + +-- | Transforms buffer chunks of a CSV file to parsed +-- | arrays of CSV values. +parseRaw :: Pipe (Maybe Buffer) (Maybe (Array String)) Aff Unit +parseRaw = do + s <- liftEffect $ CSV.Parse.toObjectStream <$> CSV.Parse.make {} + Pipes.Stream.fromTransform s + +-- | Transforms CSV rows into stringified CSV records +-- | using the given ordered array of column names. +stringifyRaw :: Array String -> Pipe (Maybe (Array String)) (Maybe String) Aff Unit +stringifyRaw columns = do + s <- liftEffect $ CSV.Stringify.toObjectStream <$> CSV.Stringify.make columns {} + Pipes.Stream.fromTransform s + +-- | Transforms purescript records into stringified CSV records. +-- | +-- | Columns are inferred from the record's keys, ordered alphabetically. +stringify :: forall r rl. WriteCSVRecord r rl => RowToList r rl => Keys rl => Pipe (Maybe { | r }) (Maybe String) Aff Unit +stringify = do + raw <- liftEffect $ CSV.Stringify.make (Array.fromFoldable $ keys $ Proxy @r) {} + let + printer = Pipes.Stream.fromTransform $ CSV.Stringify.toObjectStream raw + marshal = forever $ yield =<< (writeCSVRecord @r @rl <$> await) + Pipes.Stream.inEOS marshal >-> printer diff --git a/test/Test/Main.purs b/test/Test/Main.purs new file mode 100644 index 0000000..01eb821 --- /dev/null +++ b/test/Test/Main.purs @@ -0,0 +1,14 @@ +module Test.Main where + +import Prelude + +import Data.Maybe (Maybe(..)) +import Effect (Effect) +import Effect.Aff (launchAff_) +import Test.Pipes.CSV as Test.Pipes.CSV +import Test.Spec.Reporter (specReporter) +import Test.Spec.Runner (defaultConfig, runSpec') + +main :: Effect Unit +main = launchAff_ $ runSpec' (defaultConfig { failFast = true, timeout = Nothing }) [ specReporter ] do + Test.Pipes.CSV.spec diff --git a/test/Test/Pipes.CSV.purs b/test/Test/Pipes.CSV.purs new file mode 100644 index 0000000..d17353a --- /dev/null +++ b/test/Test/Pipes.CSV.purs @@ -0,0 +1,88 @@ +module Test.Pipes.CSV where + +import Prelude + +import Control.Monad.Gen (chooseInt) +import Control.Monad.Rec.Class (Step(..), tailRecM) +import Data.Array as Array +import Data.DateTime (DateTime) +import Data.Foldable (fold, sum) +import Data.Maybe (Maybe(..), fromJust) +import Data.Newtype (wrap) +import Data.PreciseDateTime (fromRFC3339String, toDateTimeLossy) +import Data.String.CodePoints as String.CodePoints +import Data.Tuple.Nested ((/\)) +import Effect.Class (liftEffect) +import Effect.Console (log) +import Node.Encoding (Encoding(..)) +import Partial.Unsafe (unsafePartial) +import Pipes (yield, (>->)) +import Pipes.CSV as Pipes.CSV +import Pipes.Collect as Pipes.Collect +import Pipes.Construct as Pipes.Construct +import Pipes.Node.Buffer as Pipes.Buffer +import Pipes.Node.Stream as Pipes.Stream +import Pipes.Prelude (chain, map, toListM) as Pipes +import Pipes.Util as Pipes.Util +import Test.QuickCheck.Gen (randomSample') +import Test.Spec (Spec, before, describe, it) +import Test.Spec.Assertions (shouldEqual) + +csv :: String +csv = + """created,flag,foo,id +2020-01-01T00:00:00.0Z,true,a,1 +2024-02-02T08:00:00.0Z,false,apple,2 +1970-01-01T00:00:00.0Z,true,hello,3 +""" + +dt :: String -> DateTime +dt = toDateTimeLossy <<< unsafePartial fromJust <<< fromRFC3339String <<< wrap + +spec :: Spec Unit +spec = + describe "Pipes.CSV" do + it "stringify" do + let + objs = + [ { id: 1, foo: "a", flag: true, created: dt "2020-01-01T00:00:00Z" } + , { id: 2, foo: "apple", flag: false, created: dt "2024-02-02T08:00:00Z" } + , { id: 3, foo: "hello", flag: true, created: dt "1970-01-01T00:00:00Z" } + ] + + csv' <- map fold $ Pipes.Collect.toArray $ Pipes.Stream.withEOS (Pipes.Construct.eachArray objs) >-> Pipes.CSV.stringify >-> Pipes.Stream.unEOS + csv' `shouldEqual` csv + describe "parse" do + it "parses csv" do + rows <- map Array.fromFoldable + $ Pipes.toListM + $ Pipes.Stream.withEOS (yield csv) + >-> Pipes.Stream.inEOS (Pipes.Buffer.fromString UTF8) + >-> Pipes.CSV.parse + >-> Pipes.Stream.unEOS + + rows `shouldEqual` + [ { id: 1, foo: "a", flag: true, created: dt "2020-01-01T00:00:00Z" } + , { id: 2, foo: "apple", flag: false, created: dt "2024-02-02T08:00:00Z" } + , { id: 3, foo: "hello", flag: true, created: dt "1970-01-01T00:00:00Z" } + ] + before + (do + nums <- liftEffect $ randomSample' 100000 (chooseInt 0 9) + let + chars = [ "i","d","\n" ] <> join ((\n -> [show n, "\n"]) <$> nums) + bufs <- Pipes.Collect.toArray + $ Pipes.Stream.withEOS (Pipes.Construct.eachArray chars) + >-> Pipes.Util.chunked 1000 + >-> Pipes.Stream.inEOS (Pipes.map fold >-> Pipes.Buffer.fromString UTF8) + >-> Pipes.Stream.unEOS + pure $ nums /\ bufs + ) + $ it "parses large csv" \(nums /\ bufs) -> do + rows <- + Pipes.Collect.toArray + $ Pipes.Stream.withEOS (Pipes.Construct.eachArray bufs) + >-> Pipes.CSV.parse @(id :: Int) + >-> Pipes.Stream.unEOS + + rows `shouldEqual` ((\id -> { id }) <$> nums)