1
! pip install langchain-core langchain_community langchain-openai langchain-huggingface langchain-ollama
2
! pip install wikipedia pandas numpy requests rich pyyaml rouge_score ollama

1
%env TOKENIZERS_PARALLELISM=false

1
import getpass
2
import json
3
import requests
4
5
6
def distil_bearer_token(DL_USERNAME: str, DL_PASSWORD: str) -> str:
7
    response = requests.post(
8
        "https://cognito-idp.eu-central-1.amazonaws.com",
9
        headers={
10
            "X-Amz-Target": "AWSCognitoIdentityProviderService.InitiateAuth",
11
            "Content-Type": "application/x-amz-json-1.1",
12
        },
13
        data=json.dumps({
14
            "AuthParameters": {
15
                "USERNAME": DL_USERNAME,
16
                "PASSWORD": DL_PASSWORD,
17
            },
18
            "AuthFlow": "USER_PASSWORD_AUTH",
19
            "ClientId" : "4569nvlkn8dm0iedo54nbta6fd",
20
        })
21
    )
22
    response.raise_for_status()
23
    return response.json()["AuthenticationResult"]["AccessToken"]
24
25
26
DL_USERNAME = "YOUR_EMAIL"
27
DL_PASSWORD = getpass.getpass()
28
29
AUTH_HEADER = {"Authorization": distil_bearer_token(DL_USERNAME, DL_PASSWORD)}
30
print("Success")

1
## Register a model
2
from pprint import pprint
3
4
# Register a model
5
data = {"name": "testmodel-123"}
6
response = requests.post(
7
    "https://api.distillabs.ai/models",
8
    data=json.dumps(data),
9
    headers={"content-type": "application/json", **AUTH_HEADER},
10
)
11
pprint(response.json())
12
model_id = response.json()["id"]
13
print(f"Registered a model with ID={model_id}")

1
from pprint import pprint
2
3
# Retrieve all models
4
response = requests.get(
5
    "https://api.distillabs.ai/models",
6
    headers=AUTH_HEADER
7
)
8
pprint(response.json())

1
import json
2
from pathlib import Path
3
import rich.json
4
5
with open(Path("data").joinpath("job_description.json")) as fin:
6
    rich.print(rich.json.JSON(fin.read()))

1
from pathlib import Path
2
from IPython.display import display
3
4
import pandas as pd
5
6
print("# --- Train set")
7
train = pd.read_csv(Path("data").joinpath("train.csv"))
8
display(train)
9
10
print("# --- Test set")
11
test = pd.read_csv(Path("data").joinpath("test.csv"))
12
display(test)

1
import pandas as pd
2
3
unstructured = pd.read_csv(Path("data").joinpath("unstructured.csv"))
4
display(unstructured)

1
import json
2
3
import requests
4
import yaml
5
from pathlib import Path
6
import pandas
7
8
# Specify the config
9
config = {
10
    "base": {
11
        "task": "question-answering-open-book",
12
    },
13
    "tuning": {
14
        "num_train_epochs": 4,
15
    },
16
}
17
18
# Package your data
19
data_dir = Path("data")
20
data = {
21
    "job_description": {
22
        "type": "json",
23
        "content": open(data_dir / "job_description.json", encoding="utf-8").read()
24
    },
25
    "train_data": {
26
        "type": "csv",
27
        "content": open(data_dir / "train.csv", encoding="utf-8").read()
28
    },
29
    "test_data": {
30
        "type": "csv",
31
        "content": open(data_dir / "test.csv", encoding="utf-8").read()
32
    },
33
    "unstructured_data": {
34
        "type": "csv",
35
        "content": open(data_dir / "unstructured.csv", encoding="utf-8").read()
36
    },
37
    "config": {
38
        "type": "yaml",
39
        "content": yaml.dump(config)
40
    },
41
}
42
43
# Upload data
44
response = requests.post(
45
    f"https://api.distillabs.ai/models/{model_id}/uploads",
46
    data=json.dumps(data),
47
    headers={"content-type": "application/json", **AUTH_HEADER},
48
)
49
print(response.json())
50
upload_id = response.json()["id"]

1
from pprint import pprint
2
3
# Start teacher evaluation
4
data = {"upload_id": upload_id}
5
response = requests.post(
6
    f"https://api.distillabs.ai/models/{model_id}/teacher-evaluations",
7
    data=json.dumps(data),
8
    headers={"content-type": "application/json", **AUTH_HEADER},
9
)
10
11
pprint(response.json())
12
teacher_evaluation_id = response.json().get("id")

1
from pprint import pprint
2
3
response = requests.get(
4
    f"https://api.distillabs.ai/teacher-evaluations/{teacher_evaluation_id}/status",
5
    headers=AUTH_HEADER
6
)
7
pprint(response.json())

1
import time
2
from pprint import pprint
3
4
# Start SLM training
5
data = {"upload_id": upload_id}
6
response = requests.post(
7
    f"https://api.distillabs.ai/models/{model_id}/training",
8
    data=json.dumps(data),
9
    headers={"content-type": "application/json", **AUTH_HEADER},
10
)
11
12
pprint(response.json())
13
slm_training_job_id = response.json().get("id")

1
import json
2
from pprint import pprint
3
import requests
4
5
response = requests.get(
6
    f"https://api.distillabs.ai/trainings/{slm_training_job_id}/status",
7
    headers=AUTH_HEADER,
8
)
9
pprint(response.json())

1
from pprint import pprint
2
3
response = requests.get(
4
    f"https://api.distillabs.ai/trainings/{slm_training_job_id}/evaluation-results",
5
    headers=AUTH_HEADER,
6
)
7
8
pprint(response.json())

1
import json
2
from pprint import pprint
3
import requests
4
5
response = requests.get(
6
    f"https://api.distillabs.ai/models",
7
    headers=AUTH_HEADER,
8
)
9
pprint(response.json())

1
from pprint import pprint
2
3
# Get model download URL
4
slm_training_job_id = "SELECTED-MODEL"
5
response = requests.get(
6
    f"https://api.distillabs.ai/trainings/{slm_training_job_id}/model",
7
    headers=AUTH_HEADER
8
)
9
10
s3url = response.json()["s3_url"]
11
pprint(response.json())

1
import tarfile
2
import urllib.request
3
4
print("Downloading …")
5
def status(count, block, total):
6
    print("\r", f"Downloading: {count * block / total:.1%}", end="")
7
8
9
urllib.request.urlretrieve(
10
    s3url,
11
    "model.tar",
12
    reporthook=status,
13
)
14
15
print("\nUnpacking …")
16
with tarfile.open("model.tar", mode="r:*") as tar:
17
    tar.extractall(path=".")

1
!ls -lt

1
! ollama list

1
! curl -fsSL https://ollama.com/install.sh | sh

1
! nohup ollama serve &

1
! ollama list

1
! ollama create model-distillabs -f model/Modelfile

1
from openai import OpenAI
2
3
client = OpenAI(
4
    base_url = 'http://localhost:11434/v1',
5
    api_key='ollama', # required, but unused
6
)
7
8
response = client.chat.completions.create(
9
  model="model-distillabs",
10
  messages=[
11
    {"role": "user", "content": "What day is it?"},
12
  ],
13
)
14
print(response.choices[0].message.content)

1
SYSTEM_PROMPT = """
2
You are a problem solving model working on task_description XML block:
3
4
<task_description>
5
Answer the question using information in the context. Questions require information from more than one paragraph from the context to answer.
6
</task_description>
7
8
You will be given a single task with context in the context XML block and the task in the question XML block
9
Solve the task in question block based on the context in context block.
10
Generate only the answer, do not generate anything else
11
"""
12
13
PROMPT_TEMPLATE = """
14
Now for the real task, solve the task in question block based on the context in context block.
15
Generate only the solution, do not generate anything else.
16
17
<context>
18
{context}
19
</context>
20
21
<question>
22
{question}
23
</question>
24
"""
25
26
def get_prompt(self, question: str, context: str) -> str:
27
    return [
28
        {"role": "system", "content": SYSTEM_PROMPT},
29
        {"role": "user", "content": PROMPT_TEMPLATE.format(context=context, question=question)},
30
    ]

1
import pandas
2
from pathlib import Path
3
4
data_location = Path("data")
5
test = pandas.read_csv(data_location.joinpath("test.csv"))
6
7
example = test.loc[3]
8
print(f"context:\n{example['context']}")
9
print(f"question:\n{example['question']}")
10
print(f"answer:\n{example['answer']}")
11
12
13
response = client.chat.completions.create(
14
  model="model-distillabs",
15
  messages=get_prompt(question=example['question'], context=example['context']),
16
)
17
print("\n\nPrediction:", response.choices[0].message.content)