commit d72f836cdd71ca7571d5c23288ee42099f2e686e Author: Orion Kindel Date: Sat Nov 18 14:03:35 2023 -0600 feat: initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ebf58ad --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +/bower_components/ +/node_modules/ +/.pulp-cache/ +/output/ +/generated-docs/ +/.psc-package/ +/.psc* +/.purs* +/.psa* +/.spago +.log +.purs-repl +.env diff --git a/.prettierrc.cjs b/.prettierrc.cjs new file mode 100644 index 0000000..3a11283 --- /dev/null +++ b/.prettierrc.cjs @@ -0,0 +1,8 @@ +module.exports = { + tabWidth: 2, + trailingComma: 'all', + singleQuote: true, + semi: false, + arrowParens: 'avoid', + plugins: ['prettier-plugin-sh'], +} diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..598836b --- /dev/null +++ b/.tool-versions @@ -0,0 +1,3 @@ +bun 1.0.11 +purescript 0.15.12 +nodejs 20.9.0 diff --git a/bun/fmt.js b/bun/fmt.js new file mode 100644 index 0000000..db578f1 --- /dev/null +++ b/bun/fmt.js @@ -0,0 +1,28 @@ +/** @type {(parser: string, ps: string[]) => import("bun").Subprocess} */ +const prettier = (parser, ps) => + Bun.spawn(['bun', 'x', 'prettier', '--write', '--parser', parser, ...ps], { + stdout: 'inherit', + stderr: 'inherit', + }) + +const procs = [ + prettier('babel', ['./src/**/*.js', './bun/**/*.js', './.prettierrc.cjs']), + prettier('json', ['./package.json', './jsconfig.json']), + prettier('sh', ['./Dockerfile']), + Bun.spawn( + [ + 'bun', + 'x', + 'purs-tidy', + 'format-in-place', + 'src/**/*.purs', + 'test/**/*.purs', + ], + { + stdout: 'inherit', + stderr: 'inherit', + }, + ), +] + +await Promise.all(procs.map(p => p.exited)) diff --git a/jsconfig.json b/jsconfig.json new file mode 100644 index 0000000..f48b93c --- /dev/null +++ b/jsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "types": ["bun-types"], + "lib": ["esnext"], + "target": "esnext", + "module": "esnext", + "moduleResolution": "bundler", + "moduleDetection": "force", + "jsx": "react", + "allowJs": true, + "checkJs": true, + "noEmit": true, + "strict": true + }, + "include": ["src/**/*.js", "bun/**/*.js"] +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..679d544 --- /dev/null +++ b/package.json @@ -0,0 +1,32 @@ +{ + "name": "@ts/scraper", + "private": true, + "module": "index.js", + "type": "module", + "scripts": { + "fmt": "bun bun/fmt.js", + "postinstall": "rm -rf node_modules/puppeteer-extra/node_modules/" + }, + "devDependencies": { + "bun-types": "1.0.11", + "prettier-plugin-sh": "^0.13.1", + "purs-tidy": "^0.10.0", + "spago": "next" + }, + "peerDependencies": { + "typescript": "^5.0.0" + }, + "dependencies": { + "callsites": "^4.1.0", + "decimal.js": "^10.4.3", + "nanoid": "^5.0.2", + "pg": "^8.11.3", + "puppeteer-core": "https://git.orionkindel.com/orion/-/packages/npm/puppeteer-core/0.0.2-fork/files/3271#.tgz", + "puppeteer": "https://git.orionkindel.com/orion/-/packages/npm/puppeteer/0.0.3-fork/files/3272#.tgz", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-adblocker": "^2.13.6", + "puppeteer-extra-plugin-anonymize-ua": "^2.4.6", + "puppeteer-extra-plugin-recaptcha": "^3.6.8", + "puppeteer-extra-plugin-stealth": "^2.11.2" + } +} diff --git a/spago.yaml b/spago.yaml new file mode 100644 index 0000000..f40ac42 --- /dev/null +++ b/spago.yaml @@ -0,0 +1,319 @@ +package: + dependencies: + - aff + - arraybuffer + - avar + - console + - crypto + - effect + - either + - foldable-traversable + - foreign-object + - httpurple + - nanoid + - newtype + - node-event-emitter + - node-http + - node-process + - node-streams + - node-zlib + - pipes + - posix-types + - postgresql-client + - prelude + - puppeteer + - spec + - strings + - stringutils + - sync + - transformers + - tuples + - typelevel-prelude + - unlift + - yoga-fetch + name: scraper +workspace: + extra_packages: + node-url: + dependencies: [ 'effect', 'foreign', 'nullable', 'tuples' ] + git: 'https://github.com/cakekindel/purescript-node-url.git' + ref: '2dea74c' + crypto: + dependencies: + [ + "aff", + "effect", + "functions", + "maybe", + "node-buffer", + "nullable", + "prelude" + ] + git: 'https://github.com/oreshinya/purescript-crypto.git' + ref: 'c72b4b7' + puppeteer: + dependencies: + - aff + - aff-promise + - arrays + - bifunctors + - console + - control + - datetime + - effect + - either + - enums + - exceptions + - filterable + - foldable-traversable + - foreign + - identity + - integers + - maybe + - newtype + - node-buffer + - node-path + - node-process + - node-streams + - nullable + - ordered-collections + - parallel + - prelude + - simple-json + - spec + - st + - strings + - tailrec + - transformers + - tuples + - unsafe-coerce + - web-cssom + - web-dom + - web-html + git: https://git.orionkindel.com/orion/purescript-puppeteer.git + ref: '5db9c3e' + sync: + dependencies: + - aff + - arrays + - avar + git: https://git.orionkindel.com/orion/purescript-sync.git + ref: 'f58fa86' + postgresql-client: + dependencies: + - "aff" + - "argonaut" + - "arrays" + - "assert" + - "bifunctors" + - "bytestrings" + - "datetime" + - "debug" + - "decimals" + - "dotenv" + - "effect" + - "either" + - "enums" + - "exceptions" + - "foldable-traversable" + - "foreign" + - "foreign-generic" + - "foreign-object" + - "identity" + - "integers" + - "js-date" + - "js-unsafe-stringify" + - "lists" + - "maybe" + - "newtype" + - "node-process" + - "nullable" + - "numbers" + - "ordered-collections" + - "partial" + - "polyform" + - "polyform-batteries-core" + - "polyform-batteries-env" + - "prelude" + - "profunctor" + - "psci-support" + - "record" + - "string-parsers" + - "strings" + - "test-unit" + - "transformers" + - "tuples" + - "typelevel-prelude" + - "unsafe-coerce" + - "validation" + git: https://github.com/rightfold/purescript-postgresql-client.git + ref: main + httpurple: + dependencies: + - aff + - arrays + - bifunctors + - console + - control + - debug + - effect + - either + - exceptions + - foldable-traversable + - foreign + - foreign-object + - functions + - js-uri + - justifill + - lists + - literals + - maybe + - newtype + - node-buffer + - node-event-emitter + - node-fs + - node-net + - node-http + - node-process + - node-streams + - options + - ordered-collections + - posix-types + - prelude + - profunctor + - record + - record-studio + - refs + - routing-duplex + - safe-coerce + - strings + - transformers + - tuples + - type-equality + - typelevel-prelude + - unsafe-coerce + - untagged-union + git: https://github.com/cakekindel/purescript-httpurple.git + ref: main + polyform: + dependencies: + [ + "heterogeneous", + "js-unsafe-stringify", + "newtype", + "ordered-collections", + "variant", + "profunctor", + "invariant", + "foreign-object", + "run", + "transformers", + "validation", + "foreign" + ] + git: "https://github.com/purescript-polyform/polyform.git" + ref: "v0.9.2" + polyform-batteries-core: + dependencies: + [ + "debug", + "decimals", + "filterable", + "numbers", + "polyform", + "prelude", + "record-extra", + "test-unit" + ] + git: "https://github.com/purescript-polyform/batteries-core.git" + ref: "v0.3.0" + polyform-batteries-urlencoded: + dependencies: + [ + "argonaut", + "console", + "debug", + "effect", + "form-urlencoded", + "polyform-batteries-core", + "psci-support", + "spec" + ] + git: "https://github.com/purescript-polyform/batteries-urlencoded.git" + ref: "v0.4.1" + polyform-batteries-env: + dependencies: + [ + "arrays", + "identity", + "maybe", + "ordered-collections", + "polyform", + "polyform-batteries-core", + "prelude", + "psci-support", + "typelevel-prelude" + ] + git: "https://github.com/purescript-polyform/batteries-env.git" + ref: "v0.2.0" + bytestrings: + dependencies: + [ + "arrays", + "console", + "effect", + "exceptions", + "foldable-traversable", + "integers", + "leibniz", + "maybe", + "newtype", + "node-buffer", + "partial", + "prelude", + "quickcheck", + "quickcheck-laws", + "quotient", + "unsafe-coerce" + ] + git: "https://github.com/martyall/purescript-bytestrings.git" + ref: "e51cf868a4137c1c48c98d32115bb2014c9f7624" + quotient: + dependencies: [ "prelude", "quickcheck" ] + git: "https://github.com/rightfold/purescript-quotient.git" + ref: "v3.0.0" + foreign-generic: + dependencies: + [ + "arrays", + "assert", + "bifunctors", + "console", + "control", + "effect", + "either", + "exceptions", + "foldable-traversable", + "foreign", + "foreign-object", + "identity", + "lists", + "maybe", + "newtype", + "partial", + "prelude", + "record", + "strings", + "transformers", + "tuples", + "unsafe-coerce" + ] + git: "https://github.com/paluh/purescript-foreign-generic.git" + ref: "a5c23d29e72619624978446293ac9bb45ccd2fde" + js-unsafe-stringify: + dependencies: [] + git: "https://github.com/paluh/purescript-js-unsafe-stringify.git" + ref: "master" + package_set: + url: https://raw.githubusercontent.com/purescript/package-sets/psc-0.15.10-20230930/packages.json + hash: sha256-nTsd44o7/hrTdk0c6dh0wyBqhFFDJJIeKdQU6L1zv/A= diff --git a/src/Main.purs b/src/Main.purs new file mode 100644 index 0000000..c1a9aaa --- /dev/null +++ b/src/Main.purs @@ -0,0 +1,6 @@ +module Main where + +import Prelude + +main :: Effect Unit +main = pure unit