refactor: nginx out of system, add finalize step lifecycle

This commit is contained in:
Orion Kindel 2023-06-17 12:36:00 -05:00
parent b68d3fc144
commit 8f3cb8fb09
Signed by untrusted user who does not match committer: orion
GPG Key ID: 6D4165AE4C928719
9 changed files with 321 additions and 278 deletions

View File

@ -7,8 +7,7 @@
"author": "Orion Kindel <cakekindel@gmail.com>",
"license": "MIT",
"scripts": {
"fmt:check": "prettier --check src/**/*.js test/**/*.js",
"fmt": "prettier --write src/**/*.js test/**/*.js",
"fmt": "prettier --write src/*.js test/*.js src/**/*.js test/**/*.js",
"test": "ava"
},
"dependencies": {

View File

@ -4,7 +4,7 @@ const cp = require('child_process')
const System = require('./system.js')
const run = (script, {stdio = undefined} = {}) =>
const run = (script, { stdio = undefined } = {}) =>
pipe(
Either.tryCatch(
() => cp.spawnSync(script, { encoding: 'utf8', shell: true, stdio }),

View File

@ -178,7 +178,9 @@ const parseSegmentNetwork = ({ http = true }, svc) =>
[
`network.interface is 'public' and service uses ${
http ? 'http' : 'tcp'
}; ${http ? 'network.domain' : 'network.domain and network.ip'} must be set.`,
}; ${
http ? 'network.domain' : 'network.domain and network.ip'
} must be set.`,
yaml.stringify(n),
].join('\n'),
),

View File

@ -10,32 +10,38 @@ const DbService = require('./service/db.js')
const Config = require('./config.js')
const Cmd = require('./cmd.js')
const execSteps = (res, step) => {
console.log(step.label)
return Either.flatMap(step.work)(res)
}
pipe(
Cmd.run('cat ./config.yml'),
Either.map(({stdout}) => stdout),
Either.map(({ stdout }) => stdout),
Either.flatMap(Config.parse),
Either.map(svcs =>
svcs.reduce(
(steps, svc) =>
svc.serviceType === 'db' ? [...steps, ...DbService.steps(svc)] : steps,
[System.step(svcs)],
System.steps(svcs),
),
),
Either.flatMap(steps =>
Array_.reverse(steps).reduce((res, step) => {
if (step.down) {
console.log(step.down.label)
return Either.flatMap(step.down.work)(res)
} else {
return res;
}
}, Either.right(steps)),
Array_.reverse(steps)
.filter(step => step.on === 'down')
.reduce(execSteps, Either.right(steps)),
),
Either.flatMap(steps =>
steps.reduce((res, step) => {
console.log(step.up.label)
return Either.flatMap(step.up.work)(res)
}, Either.right(steps)),
steps
.filter(step => step.on === 'up')
.reduce(execSteps, Either.right(steps)),
),
Either.getOrElse(e => {throw e;})
Either.flatMap(steps =>
steps
.filter(step => step.on === 'finalize')
.reduce(execSteps, Either.right(steps)),
),
Either.getOrElse(e => {
throw e
}),
)

View File

@ -3,4 +3,7 @@ const Cmd = require('./cmd.js')
const inaddrAny = '0.0.0.0'
const inaddrLoopback = '127.0.0.1'
module.exports = {publicInterfaceIp: inaddrAny, localInterfaceIp: inaddrLoopback}
module.exports = {
publicInterfaceIp: inaddrAny,
localInterfaceIp: inaddrLoopback,
}

252
src/nginx.js Normal file
View File

@ -0,0 +1,252 @@
const net = require('net')
const { pipe, flow } = require('fp-ts/function')
const Either = require('fp-ts/Either')
const Cmd = () => require('./cmd.js')
const rootConf = [
'worker_processes 1;',
'',
'error_log logs/error.log debug;',
`log_format basic '$remote_addr [$time_local] '`,
` '$protocol $status $bytes_sent $bytes_received '`,
` '$session_time';`,
'',
'access_log logs/access.log basic buffer=32k;',
'',
'events {',
' worker_connections 1024;',
'}',
'',
'stream {',
' include /etc/nginx/streams-enabled/*;',
'}',
'',
'http {',
' include mime.types;',
' default_type application/octet-stream;',
'',
' include /etc/nginx/sites-enabled/*;',
'',
' sendfile on;',
' keepalive_timeout 65;',
'}',
].join('\n')
const systemdUnit = [
'[Unit]',
'Description=The NGINX HTTP and reverse proxy server',
'After=syslog.target network-online.target remote-fs.target nss-lookup.target',
'Wants=network-online.target',
'',
'[Service]',
'Type=forking',
'PIDFile=/usr/local/nginx/nginx.pid',
'ExecStartPre=/usr/local/nginx/nginx -t',
'ExecStart=/usr/local/nginx/nginx',
'ExecReload=/usr/local/nginx/nginx -s reload',
'ExecStop=/bin/kill -s QUIT $MAINPID',
'PrivateTmp=true',
'',
'[Install]',
'WantedBy=multi-user.target',
].join('\n')
const confs = svcs =>
svcs
.filter(cfg => cfg.network.interface === 'public')
.map(cfg => ({
path: `/etc/nginx/${
cfg.serviceType === 'db' ? 'streams' : 'sites'
}-available/${cfg.network.domain}`,
symlinks: [
`/etc/nginx/${cfg.serviceType === 'db' ? 'streams' : 'sites'}-enabled/${
cfg.network.domain
}`,
],
contents:
cfg.serviceType === 'db'
? [
`upstream ${cfg.linuxUser.username}_pg {`,
` server localhost:${cfg.network.port};`,
'}',
'',
'server {',
` listen ${
net.isIP(cfg.network.ip) === 6
? '[' + cfg.network.ip + ']'
: cfg.network.ip
}:5432;`,
` proxy_pass ${cfg.linuxUser.username}_pg;`,
'}',
].join('\n')
: [
'server {',
` listen 80;`,
` server_name ${cfg.network.domain};`,
'',
' location / {',
' client_max_body_size 512M;',
` proxy_pass http://localhost:${cfg.network.port};`,
' proxy_set_header Host $host;',
' proxy_set_header X-Real-IP $remote_addr;',
' proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;',
' proxy_set_header X-Forwarded-Proto $scheme;',
' }',
'}',
].join('\n'),
}))
const installed = () =>
pipe(
Cmd().run('nginx -V 2>&1'),
Either.map(e => ({ shouldInstall: !e.stdout.includes('--with-stream') })),
Either.orElse(e => {
try {
const { stdout } = JSON.parse(e.message)
return stdout.includes('not found')
? Either.right({ shouldInstall: true })
: Either.left(e)
} catch (e) {
return Either.left(e)
}
}),
Either.flatMap(({ shouldInstall }) =>
!shouldInstall
? Either.right()
: Cmd().run(
[
'set -x',
'echo "nginx needs the stream module, building from source... (this will take a while)"',
'sleep 2',
'rm *.tar.gz || true',
// pcre
'wget -q github.com/PCRE2Project/pcre2/releases/download/pcre2-10.42/pcre2-10.42.tar.gz',
'tar -zxf pcre2-10.42.tar.gz',
'cd pcre2-10.42',
'./configure',
'make 1>/dev/null',
'make install 1>/dev/null',
'cd ..',
// openssl
'wget -q https://www.openssl.org/source/openssl-1.1.1u.tar.gz',
'tar -zxf openssl-1.1.1u.tar.gz',
'cd openssl-1.1.1u',
'./config --prefix=/usr 1>/dev/null',
'make 1>/dev/null',
'make install 1>/dev/null',
'cd ..',
// zlib
'wget -q http://zlib.net/zlib-1.2.13.tar.gz',
'tar -zxf zlib-1.2.13.tar.gz',
'cd zlib-1.2.13',
'./configure 1>/dev/null',
'make 1>/dev/null',
'make install 1>/dev/null',
'cd ..',
// nginx
'wget https://nginx.org/download/nginx-1.25.1.tar.gz',
'tar -zxf nginx-1.25.1.tar.gz',
'cd nginx-1.25.1',
[
'./configure',
'--sbin-path=/usr/local/nginx/nginx',
'--conf-path=/usr/local/nginx/nginx.conf',
'--pid-path=/usr/local/nginx/nginx.pid',
'--with-pcre=../pcre2-10.42',
'--with-zlib=../zlib-1.2.13',
'--with-stream',
'--with-threads',
'--with-http_ssl_module',
'--with-pcre-jit',
'--with-http_stub_status_module',
'--with-http_realip_module',
'--with-http_auth_request_module',
'--with-http_v2_module',
'--with-http_dav_module',
'--with-http_slice_module',
'--with-http_addition_module',
'--with-http_gunzip_module',
'--with-http_gzip_static_module',
'--with-http_sub_module',
'--with-debug',
'1>/dev/null',
].join(' '),
'make 1>/dev/null',
'make install 1>/dev/null',
// nginx.conf
'cat <<"NGINXCONF" > /usr/local/nginx/nginx.conf',
rootConf,
'NGINXCONF',
// systemd
'cat <<"NGINXUNIT" > /lib/systemd/system/nginx.service',
systemdUnit,
'NGINXUNIT',
'systemctl enable nginx',
'rm /usr/sbin/nginx || true',
'ln /usr/local/nginx/nginx /usr/sbin/nginx',
].join('\n'),
{ stdio: 'inherit' },
),
),
)
const confsWritten = svcs =>
confs(svcs).reduce(
(res, { path, symlinks, contents }) =>
Either.flatMap(() =>
Cmd().run(
[
`cat <<"EOCONF" > ${path}`,
contents,
'EOCONF',
...symlinks.map(ln => `ln -s ${path} ${ln}`),
].join('\n'),
),
)(res),
Either.right(undefined),
)
const confsDeleted = svcs =>
confs(svcs).reduce(
(res, { path, symlinks }) =>
Either.flatMap(() =>
Cmd().run(
[
`rm ${path} || true`,
...symlinks.map(ln => `rm ${ln} || true`),
].join('\n'),
),
)(res),
Either.right(undefined),
)
const started = () => Cmd().run('systemctl restart nginx')
const steps = svcs => [
{
on: 'up',
label: 'nginx: setup',
work: flow(
Either.right,
Either.tap(installed),
Either.tap(() => confsWritten(svcs)),
Either.tap(started),
),
},
{
on: 'down',
label: 'nginx: teardown',
work: flow(
Either.right,
Either.tap(() => confsDeleted(svcs)),
),
},
{
on: 'finalize',
label: 'nginx: cleanup',
work: flow(Either.right, Either.tap(started)),
},
]
module.exports = { steps }

View File

@ -24,26 +24,24 @@ const dockerCompose = cfg =>
})
const steps = cfg => [
LinuxUser.step(cfg),
...LinuxUser.steps(cfg),
{
up: {
label: `${cfg.linuxUser.username}: create db on ${cfg.network.interfaceIp}:${cfg.network.port}`,
work: a =>
pipe(
Cmd.doas(
cfg.linuxUser.username,
[
'set -x',
'cat << "EOCOMPOSE" > ~/docker-compose.yaml',
dockerCompose(cfg),
'EOCOMPOSE',
'docker compose up -d',
].join('\n'),
),
Either.map(() => a),
on: 'up',
label: `${cfg.linuxUser.username}: create db on ${cfg.network.interfaceIp}:${cfg.network.port}`,
work: a =>
pipe(
Cmd.doas(
cfg.linuxUser.username,
[
'set -x',
'cat << "EOCOMPOSE" > ~/docker-compose.yaml',
dockerCompose(cfg),
'EOCOMPOSE',
'docker compose up -d',
].join('\n'),
),
},
// down handled by LinuxUser killing user processes
Either.map(() => a),
),
},
]

View File

@ -31,7 +31,6 @@ const persistedStateRestored = cfg =>
` chown -R ${cfg.linuxUser.username}:${cfg.linuxUser.username} ${cfg.linuxUser.homeDir}`,
`fi`,
].join('\n'),
{ stdio: 'inherit' },
)
const sshAllowed = cfg =>
@ -94,7 +93,6 @@ const statePersisted = cfg =>
p => `mv ${p} ${path.join(cfg.linuxUser.persistDir, p)} || true`,
),
].join('\n'),
{ stdio: 'inherit' },
),
)
@ -106,8 +104,9 @@ const userDeleted = cfg =>
Either.flatMap(() => Cmd.run(`rm -r ${cfg.linuxUser.homeDir} || true`)),
)
const step = cfg => ({
up: {
const steps = cfg => [
{
on: 'up',
label: `${cfg.linuxUser.username}: create user`,
work: a =>
pipe(
@ -122,7 +121,8 @@ const step = cfg => ({
Either.map(() => a),
),
},
down: {
{
on: 'down',
label: `${cfg.linuxUser.username}: remove user`,
work: a =>
pipe(
@ -135,6 +135,6 @@ const step = cfg => ({
Either.map(() => a),
),
},
})
]
module.exports = { step }
module.exports = { steps }

View File

@ -1,56 +1,8 @@
const net = require('net')
const Either = require('fp-ts/Either')
const { pipe, flow } = require('fp-ts/function')
const Cmd = () => require('./cmd.js')
const nginxConf = [
'worker_processes 1;',
'',
'error_log logs/error.log debug;',
`log_format basic '$remote_addr [$time_local] '`,
` '$protocol $status $bytes_sent $bytes_received '`,
` '$session_time';`,
'',
'access_log logs/access.log basic buffer=32k;',
'',
'events {',
' worker_connections 1024;',
'}',
'',
'stream {',
' include /etc/nginx/streams-enabled/*;',
'}',
'',
'http {',
' include mime.types;',
' default_type application/octet-stream;',
'',
' include /etc/nginx/sites-enabled/*;',
'',
' sendfile on;',
' keepalive_timeout 65;',
'}',
].join('\n')
const nginxSystemdUnit = [
'[Unit]',
'Description=The NGINX HTTP and reverse proxy server',
'After=syslog.target network-online.target remote-fs.target nss-lookup.target',
'Wants=network-online.target',
'',
'[Service]',
'Type=forking',
'PIDFile=/usr/local/nginx/nginx.pid',
'ExecStartPre=/usr/local/nginx/nginx -t',
'ExecStart=/usr/local/nginx/nginx',
'ExecReload=/usr/local/nginx/nginx -s reload',
'ExecStop=/bin/kill -s QUIT $MAINPID',
'PrivateTmp=true',
'',
'[Install]',
'WantedBy=multi-user.target',
].join('\n')
const Nginx = require('./nginx.js')
const localSshKeyPath = '/root/.ssh/id_ed25519'
const localSshPubKeyPath = '/root/.ssh/id_ed25519.pub'
@ -104,7 +56,7 @@ const localSshKeyCreated = () =>
const localSshKeyDeleted = () => Cmd().run(`rm ${localSshKeyPath} || true`)
const firewallUp = cfgs =>
const firewallUp = svcs =>
Cmd().run(
[
'ufw default deny incoming',
@ -112,8 +64,10 @@ const firewallUp = cfgs =>
'ufw status verbose',
'ufw allow ssh',
`ufw allow 'Nginx Full'`,
...cfgs
.filter(svc => svc.serviceType === 'db' && svc.network.interface === 'public')
...svcs
.filter(
svc => svc.serviceType === 'db' && svc.network.interface === 'public',
)
.map(
svc =>
`ufw allow in to '${svc.network.ip}' port 5432 proto tcp comment ${svc.linuxUser.username}`,
@ -122,196 +76,25 @@ const firewallUp = cfgs =>
].join('\n'),
)
const nginxConfs = cfgs =>
cfgs
.filter(cfg => cfg.network.interface === 'public')
.map(cfg => ({
path: `/etc/nginx/${
cfg.serviceType === 'db' ? 'streams' : 'sites'
}-available/${cfg.network.domain}`,
symlinks: [
`/etc/nginx/${cfg.serviceType === 'db' ? 'streams' : 'sites'}-enabled/${
cfg.network.domain
}`,
],
contents:
cfg.serviceType === 'db'
? [
`upstream ${cfg.linuxUser.username}_pg {`,
` server localhost:${cfg.network.port};`,
'}',
'',
'server {',
` listen ${net.isIP(cfg.network.ip) === 6 ? '[' + cfg.network.ip + ']' : cfg.network.ip}:5432;`,
` proxy_pass ${cfg.linuxUser.username}_pg;`,
'}',
].join('\n')
: [
'server {',
` listen 80;`,
` server_name ${cfg.network.domain};`,
'',
' location / {',
' client_max_body_size 512M;',
` proxy_pass http://localhost:${cfg.network.port};`,
' proxy_set_header Host $host;',
' proxy_set_header X-Real-IP $remote_addr;',
' proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;',
' proxy_set_header X-Forwarded-Proto $scheme;',
' }',
'}',
].join('\n'),
}))
const nginxInstalled = () =>
pipe(
Cmd().run('nginx -V 2>&1'),
Either.map(e => ({ shouldInstall: !e.stdout.includes('--with-stream') })),
Either.orElse(e => {
try {
const { stdout } = JSON.parse(e.message)
return stdout.includes('not found')
? Either.right({ shouldInstall: true })
: Either.left(e)
} catch (e) {
return Either.left(e)
}
}),
Either.flatMap(({ shouldInstall }) =>
!shouldInstall
? Either.right()
: Cmd().run(
[
'set -x',
'echo "nginx needs the stream module, building from source... (this will take a while)"',
'sleep 2',
'rm *.tar.gz || true',
// pcre
'wget -q github.com/PCRE2Project/pcre2/releases/download/pcre2-10.42/pcre2-10.42.tar.gz',
'tar -zxf pcre2-10.42.tar.gz',
'cd pcre2-10.42',
'./configure',
'make 1>/dev/null',
'make install 1>/dev/null',
'cd ..',
// openssl
'wget -q https://www.openssl.org/source/openssl-1.1.1u.tar.gz',
'tar -zxf openssl-1.1.1u.tar.gz',
'cd openssl-1.1.1u',
'./config --prefix=/usr 1>/dev/null',
'make 1>/dev/null',
'make install 1>/dev/null',
'cd ..',
// zlib
'wget -q http://zlib.net/zlib-1.2.13.tar.gz',
'tar -zxf zlib-1.2.13.tar.gz',
'cd zlib-1.2.13',
'./configure 1>/dev/null',
'make 1>/dev/null',
'make install 1>/dev/null',
'cd ..',
// nginx
'wget https://nginx.org/download/nginx-1.25.1.tar.gz',
'tar -zxf nginx-1.25.1.tar.gz',
'cd nginx-1.25.1',
[
'./configure',
'--sbin-path=/usr/local/nginx/nginx',
'--conf-path=/usr/local/nginx/nginx.conf',
'--pid-path=/usr/local/nginx/nginx.pid',
'--with-pcre=../pcre2-10.42',
'--with-zlib=../zlib-1.2.13',
'--with-stream',
'--with-threads',
'--with-http_ssl_module',
'--with-pcre-jit',
'--with-http_stub_status_module',
'--with-http_realip_module',
'--with-http_auth_request_module',
'--with-http_v2_module',
'--with-http_dav_module',
'--with-http_slice_module',
'--with-http_addition_module',
'--with-http_gunzip_module',
'--with-http_gzip_static_module',
'--with-http_sub_module',
'--with-debug',
'1>/dev/null',
].join(' '),
'make 1>/dev/null',
'make install 1>/dev/null',
// nginx.conf
'cat <<"NGINXCONF" > /usr/local/nginx/nginx.conf',
nginxConf,
'NGINXCONF',
// systemd
'cat <<"NGINXUNIT" > /lib/systemd/system/nginx.service',
nginxSystemdUnit,
'NGINXUNIT',
'systemctl enable nginx',
'rm /usr/sbin/nginx || true',
'ln /usr/local/nginx/nginx /usr/sbin/nginx',
].join('\n'),
{ stdio: 'inherit' },
),
),
)
const nginxConfsCreated = cfgs =>
nginxConfs(cfgs).reduce(
(res, { path, symlinks, contents }) =>
Either.flatMap(() =>
Cmd().run(
[
`cat <<"EOCONF" > ${path}`,
contents,
'EOCONF',
...symlinks.map(ln => `ln -s ${path} ${ln}`),
].join('\n'),
),
)(res),
Either.right(undefined),
)
const nginxConfsDeleted = cfgs =>
nginxConfs(cfgs).reduce(
(res, { path, symlinks }) =>
Either.flatMap(() =>
Cmd().run(
[
`rm ${path} || true`,
...symlinks.map(ln => `rm ${ln} || true`),
].join('\n'),
),
)(res),
Either.right(undefined),
)
const nginxStarted = () => Cmd().run('systemctl restart nginx')
const step = cfgs => ({
up: {
const steps = svcs => [
{
on: 'up',
label: 'system: setup',
work: flow(
Either.right,
Either.tap(packagesInstalled),
Either.tap(dockerInstalled),
Either.tap(() => firewallUp(cfgs)),
Either.tap(() => firewallUp(svcs)),
Either.tap(localSshKeyCreated),
Either.tap(nginxInstalled),
Either.tap(() => nginxConfsCreated(cfgs)),
Either.tap(nginxStarted),
// Either.tap(sslCertsInstalled),
),
},
down: {
{
on: 'down',
label: 'system: teardown',
work: flow(
Either.right,
Either.tap(localSshKeyDeleted),
Either.tap(() => nginxConfsDeleted(cfgs)),
),
work: flow(Either.right, Either.tap(localSshKeyDeleted)),
},
})
...Nginx.steps(svcs),
]
module.exports = { step, localSshKeyPath, localSshPubKeyPath }
module.exports = { steps, localSshKeyPath, localSshPubKeyPath }