From 48bded9bf1f8c7d9bfa22c2e294f0f8d6a3e6a5a Mon Sep 17 00:00:00 2001 From: Orion Kindel Date: Mon, 18 Sep 2023 12:59:41 -0500 Subject: [PATCH] feat: initial commit --- .dockerignore | 2 ++ .gitignore | 3 +++ .tool-versions | 1 + Dockerfile | 23 +++++++++++++++++++++++ README.md | 11 +++++++++++ __main__.py | 43 +++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 7 +++++++ 7 files changed, 90 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 .tool-versions create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 __main__.py create mode 100644 requirements.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..7bd80db --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.env +env diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..405099e --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.env +model +__pycache__ diff --git a/.tool-versions b/.tool-versions new file mode 100644 index 0000000..47cd22e --- /dev/null +++ b/.tool-versions @@ -0,0 +1 @@ +python 3.10.13 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4eb4c93 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.10-bookworm AS base + +FROM base AS tini + +ENV TINI_VERSION v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] + +FROM tini AS tools +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + export DEBIAN_FRONTEND=noninteractive \ + && apt-get update -y \ + && apt-get upgrade -y + +COPY requirements.txt requirements.txt +RUN pip install -r requirements.txt + +FROM tools AS api + +COPY . . +CMD ["python", "."] diff --git a/README.md b/README.md new file mode 100644 index 0000000..43f01b0 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# Open LLaMA + +Run [OpenLLaMA](https://github.com/openlm-research/open_llama) in a GPU environment with a single command. 📡 + +## Speed Run + +1. Signup for [Beam](http://beam.cloud) +2. Download the CLI and Python SDK +3. Clone this template locally: `beam create-app openllama` +4. Spin up a GPU environment to run inference: `beam start app.py` +5. Deploy the app as a web API: `beam deploy app.py` \ No newline at end of file diff --git a/__main__.py b/__main__.py new file mode 100644 index 0000000..ad46776 --- /dev/null +++ b/__main__.py @@ -0,0 +1,43 @@ +from bottle import request, response, post, template, HTTPResponse, run +from dotenv import load_dotenv +from os import getenv + +import torch +from transformers import pipeline, T5Tokenizer, T5ForConditionalGeneration + +tokenizer = T5Tokenizer.from_pretrained( + "google/flan-t5-xl", + cache_dir = "model", +) + +model = T5ForConditionalGeneration.from_pretrained( + "google/flan-t5-xl", +) + +load_dotenv() +api_keys = getenv('API_KEYS').split(sep=",") + +generator = pipeline("text2text-generation", model=model, tokenizer=tokenizer) + +@post('/') +def gen(): + auth = request.get_header('authorization') + if auth is None: + raise HTTPResponse(status = 401) + + scheme, val = auth.split(sep=" ") + + if scheme != 'X-Api-Key' or val not in api_keys: + raise HTTPResponse(status = 401) + + body = request.json + + input = body["input"] + output = generator(input) + + print(input) + print(output) + return {"output": output[0]["generated_text"]} + +if __name__ == "__main__": + run(host='0.0.0.0', port=9010) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1d066a1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +python-dotenv~=1.0.0 +torch +accelerate +sentencepiece +protobuf +transformers +bottle